blob: 46ccd2f9737707db65451c430ff771034361458a [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/mm/page_alloc.c
3 *
4 * Manages the free list, the system allocates free pages here.
5 * Note that kmalloc() lives in slab.c
6 *
7 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
8 * Swap reorganised 29.12.95, Stephen Tweedie
9 * Support of BIGMEM added by Gerhard Wichert, Siemens AG, July 1999
10 * Reshaped it to be a zoned allocator, Ingo Molnar, Red Hat, 1999
11 * Discontiguous memory support, Kanoj Sarcar, SGI, Nov 1999
12 * Zone balancing, Kanoj Sarcar, SGI, Jan 2000
13 * Per cpu hot/cold page lists, bulk allocation, Martin J. Bligh, Sept 2002
14 * (lots of bits borrowed from Ingo Molnar & Andrew Morton)
15 */
16
Linus Torvalds1da177e2005-04-16 15:20:36 -070017#include <linux/stddef.h>
18#include <linux/mm.h>
19#include <linux/swap.h>
20#include <linux/interrupt.h>
21#include <linux/pagemap.h>
KOSAKI Motohiro10ed2732008-03-04 14:28:32 -080022#include <linux/jiffies.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023#include <linux/bootmem.h>
Yinghai Luedbe7d22010-08-25 13:39:16 -070024#include <linux/memblock.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070025#include <linux/compiler.h>
Randy Dunlap9f158332005-09-13 01:25:16 -070026#include <linux/kernel.h>
Vegard Nossumb1eeab62008-11-25 16:55:53 +010027#include <linux/kmemcheck.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070028#include <linux/module.h>
29#include <linux/suspend.h>
30#include <linux/pagevec.h>
31#include <linux/blkdev.h>
32#include <linux/slab.h>
Dave Hansena238ab52011-05-24 17:12:16 -070033#include <linux/ratelimit.h>
David Rientjes5a3135c2007-10-16 23:25:53 -070034#include <linux/oom.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070035#include <linux/notifier.h>
36#include <linux/topology.h>
37#include <linux/sysctl.h>
38#include <linux/cpu.h>
39#include <linux/cpuset.h>
Dave Hansenbdc8cb92005-10-29 18:16:53 -070040#include <linux/memory_hotplug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070041#include <linux/nodemask.h>
42#include <linux/vmalloc.h>
KOSAKI Motohiroa6cccdc2011-05-24 17:11:33 -070043#include <linux/vmstat.h>
Christoph Lameter4be38e32006-01-06 00:11:17 -080044#include <linux/mempolicy.h>
Yasunori Goto68113782006-06-23 02:03:11 -070045#include <linux/stop_machine.h>
Mel Gormanc7132162006-09-27 01:49:43 -070046#include <linux/sort.h>
47#include <linux/pfn.h>
Andrew Morton3fcfab12006-10-19 23:28:16 -070048#include <linux/backing-dev.h>
Akinobu Mita933e3122006-12-08 02:39:45 -080049#include <linux/fault-inject.h>
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -070050#include <linux/page-isolation.h>
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -070051#include <linux/page_cgroup.h>
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -070052#include <linux/debugobjects.h>
Catalin Marinasdbb1f812009-06-11 13:23:19 +010053#include <linux/kmemleak.h>
Robert Jennings925cc712009-12-17 14:44:38 +000054#include <linux/memory.h>
Mel Gorman56de7262010-05-24 14:32:30 -070055#include <linux/compaction.h>
Mel Gorman0d3d0622009-09-21 17:02:44 -070056#include <trace/events/kmem.h>
Wu Fengguang718a3822010-03-10 15:20:43 -080057#include <linux/ftrace_event.h>
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -070058#include <linux/memcontrol.h>
Linus Torvalds268bb0c2011-05-20 12:50:29 -070059#include <linux/prefetch.h>
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +010060#include <linux/migrate.h>
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -080061#include <linux/page-debug-flags.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070062
63#include <asm/tlbflush.h>
Andrew Mortonac924c62006-05-15 09:43:59 -070064#include <asm/div64.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070065#include "internal.h"
66
Lee Schermerhorn72812012010-05-26 14:44:56 -070067#ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
68DEFINE_PER_CPU(int, numa_node);
69EXPORT_PER_CPU_SYMBOL(numa_node);
70#endif
71
Lee Schermerhorn7aac7892010-05-26 14:45:00 -070072#ifdef CONFIG_HAVE_MEMORYLESS_NODES
73/*
74 * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
75 * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
76 * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem()
77 * defined in <linux/topology.h>.
78 */
79DEFINE_PER_CPU(int, _numa_mem_); /* Kernel "local memory" node */
80EXPORT_PER_CPU_SYMBOL(_numa_mem_);
81#endif
82
Linus Torvalds1da177e2005-04-16 15:20:36 -070083/*
Christoph Lameter13808912007-10-16 01:25:27 -070084 * Array of node states.
Linus Torvalds1da177e2005-04-16 15:20:36 -070085 */
Christoph Lameter13808912007-10-16 01:25:27 -070086nodemask_t node_states[NR_NODE_STATES] __read_mostly = {
87 [N_POSSIBLE] = NODE_MASK_ALL,
88 [N_ONLINE] = { { [0] = 1UL } },
89#ifndef CONFIG_NUMA
90 [N_NORMAL_MEMORY] = { { [0] = 1UL } },
91#ifdef CONFIG_HIGHMEM
92 [N_HIGH_MEMORY] = { { [0] = 1UL } },
93#endif
94 [N_CPU] = { { [0] = 1UL } },
95#endif /* NUMA */
96};
97EXPORT_SYMBOL(node_states);
98
Ravikiran G Thirumalai6c231b72005-09-06 15:17:45 -070099unsigned long totalram_pages __read_mostly;
Hideo AOKIcb45b0e2006-04-10 22:52:59 -0700100unsigned long totalreserve_pages __read_mostly;
Johannes Weinerab8fabd2012-01-10 15:07:42 -0800101/*
102 * When calculating the number of globally allowed dirty pages, there
103 * is a certain number of per-zone reserves that should not be
104 * considered dirtyable memory. This is the sum of those reserves
105 * over all existing zones that contribute dirtyable memory.
106 */
107unsigned long dirty_balance_reserve __read_mostly;
108
Jack Cheung59f9f1c2011-11-29 16:52:49 -0800109#ifdef CONFIG_FIX_MOVABLE_ZONE
110unsigned long total_unmovable_pages __read_mostly;
111#endif
Rohit Seth8ad4b1f2006-01-08 01:00:40 -0800112int percpu_pagelist_fraction;
Benjamin Herrenschmidtdcce2842009-06-18 13:24:12 +1000113gfp_t gfp_allowed_mask __read_mostly = GFP_BOOT_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800115#ifdef CONFIG_PM_SLEEP
116/*
117 * The following functions are used by the suspend/hibernate code to temporarily
118 * change gfp_allowed_mask in order to avoid using I/O during memory allocations
119 * while devices are suspended. To avoid races with the suspend/hibernate code,
120 * they should always be called with pm_mutex held (gfp_allowed_mask also should
121 * only be modified with pm_mutex held, unless the suspend/hibernate code is
122 * guaranteed not to run in parallel with that modification).
123 */
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100124
125static gfp_t saved_gfp_mask;
126
127void pm_restore_gfp_mask(void)
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800128{
129 WARN_ON(!mutex_is_locked(&pm_mutex));
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100130 if (saved_gfp_mask) {
131 gfp_allowed_mask = saved_gfp_mask;
132 saved_gfp_mask = 0;
133 }
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800134}
135
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100136void pm_restrict_gfp_mask(void)
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800137{
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800138 WARN_ON(!mutex_is_locked(&pm_mutex));
Rafael J. Wysockic9e664f2010-12-03 22:57:45 +0100139 WARN_ON(saved_gfp_mask);
140 saved_gfp_mask = gfp_allowed_mask;
141 gfp_allowed_mask &= ~GFP_IOFS;
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800142}
Mel Gormanf90ac392012-01-10 15:07:15 -0800143
144bool pm_suspended_storage(void)
145{
146 if ((gfp_allowed_mask & GFP_IOFS) == GFP_IOFS)
147 return false;
148 return true;
149}
Rafael J. Wysocki452aa692010-03-05 13:42:13 -0800150#endif /* CONFIG_PM_SLEEP */
151
Mel Gormand9c23402007-10-16 01:26:01 -0700152#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
153int pageblock_order __read_mostly;
154#endif
155
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800156static void __free_pages_ok(struct page *page, unsigned int order);
David Howellsa226f6c2006-01-06 00:11:08 -0800157
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158/*
159 * results with 256, 32 in the lowmem_reserve sysctl:
160 * 1G machine -> (16M dma, 800M-16M normal, 1G-800M high)
161 * 1G machine -> (16M dma, 784M normal, 224M high)
162 * NORMAL allocation will leave 784M/256 of ram reserved in the ZONE_DMA
163 * HIGHMEM allocation will leave 224M/32 of ram reserved in ZONE_NORMAL
164 * HIGHMEM allocation will (224M+784M)/256 of ram reserved in ZONE_DMA
Andi Kleena2f1b422005-11-05 17:25:53 +0100165 *
166 * TBD: should special case ZONE_DMA32 machines here - in those we normally
167 * don't need any ZONE_NORMAL reservation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168 */
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700169int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1] = {
Christoph Lameter4b51d662007-02-10 01:43:10 -0800170#ifdef CONFIG_ZONE_DMA
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700171 256,
Christoph Lameter4b51d662007-02-10 01:43:10 -0800172#endif
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700173#ifdef CONFIG_ZONE_DMA32
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700174 256,
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700175#endif
Christoph Lametere53ef382006-09-25 23:31:14 -0700176#ifdef CONFIG_HIGHMEM
Mel Gorman2a1e2742007-07-17 04:03:12 -0700177 32,
Christoph Lametere53ef382006-09-25 23:31:14 -0700178#endif
Mel Gorman2a1e2742007-07-17 04:03:12 -0700179 32,
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700180};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181
182EXPORT_SYMBOL(totalram_pages);
Jack Cheung59f9f1c2011-11-29 16:52:49 -0800183#ifdef CONFIG_FIX_MOVABLE_ZONE
184EXPORT_SYMBOL(total_unmovable_pages);
185#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186
Helge Deller15ad7cd2006-12-06 20:40:36 -0800187static char * const zone_names[MAX_NR_ZONES] = {
Christoph Lameter4b51d662007-02-10 01:43:10 -0800188#ifdef CONFIG_ZONE_DMA
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700189 "DMA",
Christoph Lameter4b51d662007-02-10 01:43:10 -0800190#endif
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700191#ifdef CONFIG_ZONE_DMA32
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700192 "DMA32",
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700193#endif
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700194 "Normal",
Christoph Lametere53ef382006-09-25 23:31:14 -0700195#ifdef CONFIG_HIGHMEM
Mel Gorman2a1e2742007-07-17 04:03:12 -0700196 "HighMem",
Christoph Lametere53ef382006-09-25 23:31:14 -0700197#endif
Mel Gorman2a1e2742007-07-17 04:03:12 -0700198 "Movable",
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700199};
200
Linus Torvalds1da177e2005-04-16 15:20:36 -0700201int min_free_kbytes = 1024;
Arve Hjønnevågcde89492009-02-17 14:51:02 -0800202int min_free_order_shift = 1;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700203
Jan Beulich2c85f512009-09-21 17:03:07 -0700204static unsigned long __meminitdata nr_kernel_pages;
205static unsigned long __meminitdata nr_all_pages;
Yasunori Gotoa3142c82007-05-08 00:23:07 -0700206static unsigned long __meminitdata dma_reserve;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700207
Tejun Heo0ee332c2011-12-08 10:22:09 -0800208#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
209static unsigned long __meminitdata arch_zone_lowest_possible_pfn[MAX_NR_ZONES];
210static unsigned long __meminitdata arch_zone_highest_possible_pfn[MAX_NR_ZONES];
211static unsigned long __initdata required_kernelcore;
212static unsigned long __initdata required_movablecore;
213static unsigned long __meminitdata zone_movable_pfn[MAX_NUMNODES];
Mel Gormanc7132162006-09-27 01:49:43 -0700214
Tejun Heo0ee332c2011-12-08 10:22:09 -0800215/* movable_zone is the "real" zone pages in ZONE_MOVABLE are taken from */
216int movable_zone;
217EXPORT_SYMBOL(movable_zone);
218#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
Mel Gormanc7132162006-09-27 01:49:43 -0700219
Miklos Szeredi418508c2007-05-23 13:57:55 -0700220#if MAX_NUMNODES > 1
221int nr_node_ids __read_mostly = MAX_NUMNODES;
Christoph Lameter62bc62a2009-06-16 15:32:15 -0700222int nr_online_nodes __read_mostly = 1;
Miklos Szeredi418508c2007-05-23 13:57:55 -0700223EXPORT_SYMBOL(nr_node_ids);
Christoph Lameter62bc62a2009-06-16 15:32:15 -0700224EXPORT_SYMBOL(nr_online_nodes);
Miklos Szeredi418508c2007-05-23 13:57:55 -0700225#endif
226
Mel Gorman9ef9acb2007-10-16 01:25:54 -0700227int page_group_by_mobility_disabled __read_mostly;
228
Mel Gormanb2a0ac82007-10-16 01:25:48 -0700229static void set_pageblock_migratetype(struct page *page, int migratetype)
230{
Mel Gorman49255c62009-06-16 15:31:58 -0700231
232 if (unlikely(page_group_by_mobility_disabled))
233 migratetype = MIGRATE_UNMOVABLE;
234
Mel Gormanb2a0ac82007-10-16 01:25:48 -0700235 set_pageblock_flags_group(page, (unsigned long)migratetype,
236 PB_migrate, PB_migrate_end);
237}
238
Rafael J. Wysocki7f33d492009-06-16 15:32:41 -0700239bool oom_killer_disabled __read_mostly;
240
Nick Piggin13e74442006-01-06 00:10:58 -0800241#ifdef CONFIG_DEBUG_VM
Dave Hansenc6a57e12005-10-29 18:16:52 -0700242static int page_outside_zone_boundaries(struct zone *zone, struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243{
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700244 int ret = 0;
245 unsigned seq;
246 unsigned long pfn = page_to_pfn(page);
Dave Hansenc6a57e12005-10-29 18:16:52 -0700247
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700248 do {
249 seq = zone_span_seqbegin(zone);
250 if (pfn >= zone->zone_start_pfn + zone->spanned_pages)
251 ret = 1;
252 else if (pfn < zone->zone_start_pfn)
253 ret = 1;
254 } while (zone_span_seqretry(zone, seq));
255
256 return ret;
Dave Hansenc6a57e12005-10-29 18:16:52 -0700257}
258
259static int page_is_consistent(struct zone *zone, struct page *page)
260{
Andy Whitcroft14e07292007-05-06 14:49:14 -0700261 if (!pfn_valid_within(page_to_pfn(page)))
Dave Hansenc6a57e12005-10-29 18:16:52 -0700262 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700263 if (zone != page_zone(page))
Dave Hansenc6a57e12005-10-29 18:16:52 -0700264 return 0;
265
266 return 1;
267}
268/*
269 * Temporary debugging check for pages not lying within a given zone.
270 */
271static int bad_range(struct zone *zone, struct page *page)
272{
273 if (page_outside_zone_boundaries(zone, page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700274 return 1;
Dave Hansenc6a57e12005-10-29 18:16:52 -0700275 if (!page_is_consistent(zone, page))
276 return 1;
277
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278 return 0;
279}
Nick Piggin13e74442006-01-06 00:10:58 -0800280#else
281static inline int bad_range(struct zone *zone, struct page *page)
282{
283 return 0;
284}
285#endif
286
Nick Piggin224abf92006-01-06 00:11:11 -0800287static void bad_page(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288{
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800289 static unsigned long resume;
290 static unsigned long nr_shown;
291 static unsigned long nr_unshown;
292
Wu Fengguang2a7684a2009-09-16 11:50:12 +0200293 /* Don't complain about poisoned pages */
294 if (PageHWPoison(page)) {
Andrea Arcangelief2b4b92011-03-18 00:16:35 +0100295 reset_page_mapcount(page); /* remove PageBuddy */
Wu Fengguang2a7684a2009-09-16 11:50:12 +0200296 return;
297 }
298
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800299 /*
300 * Allow a burst of 60 reports, then keep quiet for that minute;
301 * or allow a steady drip of one report per second.
302 */
303 if (nr_shown == 60) {
304 if (time_before(jiffies, resume)) {
305 nr_unshown++;
306 goto out;
307 }
308 if (nr_unshown) {
Hugh Dickins1e9e6362009-01-06 14:40:13 -0800309 printk(KERN_ALERT
310 "BUG: Bad page state: %lu messages suppressed\n",
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800311 nr_unshown);
312 nr_unshown = 0;
313 }
314 nr_shown = 0;
315 }
316 if (nr_shown++ == 0)
317 resume = jiffies + 60 * HZ;
318
Hugh Dickins1e9e6362009-01-06 14:40:13 -0800319 printk(KERN_ALERT "BUG: Bad page state in process %s pfn:%05lx\n",
Hugh Dickins3dc14742009-01-06 14:40:08 -0800320 current->comm, page_to_pfn(page));
Wu Fengguang718a3822010-03-10 15:20:43 -0800321 dump_page(page);
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700322
Dave Jones4f318882011-10-31 17:07:24 -0700323 print_modules();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324 dump_stack();
Hugh Dickinsd936cf92009-01-06 14:40:12 -0800325out:
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800326 /* Leave bad fields for debug, except PageBuddy could make trouble */
Andrea Arcangelief2b4b92011-03-18 00:16:35 +0100327 reset_page_mapcount(page); /* remove PageBuddy */
Randy Dunlap9f158332005-09-13 01:25:16 -0700328 add_taint(TAINT_BAD_PAGE);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700329}
330
Linus Torvalds1da177e2005-04-16 15:20:36 -0700331/*
332 * Higher-order pages are called "compound pages". They are structured thusly:
333 *
334 * The first PAGE_SIZE page is called the "head page".
335 *
336 * The remaining PAGE_SIZE pages are called "tail pages".
337 *
Wang Sheng-Hui6416b9f2011-11-17 10:53:50 +0100338 * All pages have PG_compound set. All tail pages have their ->first_page
339 * pointing at the head page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340 *
Hugh Dickins41d78ba2006-02-14 13:52:58 -0800341 * The first tail page's ->lru.next holds the address of the compound page's
342 * put_page() function. Its ->lru.prev holds the order of allocation.
343 * This usage means that zero-order pages may not be compound.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700344 */
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800345
346static void free_compound_page(struct page *page)
347{
Christoph Lameterd85f3382007-05-06 14:49:39 -0700348 __free_pages_ok(page, compound_order(page));
Hugh Dickinsd98c7a02006-02-14 13:52:59 -0800349}
350
Andi Kleen01ad1c02008-07-23 21:27:46 -0700351void prep_compound_page(struct page *page, unsigned long order)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352{
353 int i;
354 int nr_pages = 1 << order;
355
Andy Whitcroft33f2ef82006-12-06 20:33:32 -0800356 set_compound_page_dtor(page, free_compound_page);
Christoph Lameterd85f3382007-05-06 14:49:39 -0700357 set_compound_order(page, order);
Christoph Lameter6d777952007-05-06 14:49:40 -0700358 __SetPageHead(page);
Andy Whitcroft18229df2008-11-06 12:53:27 -0800359 for (i = 1; i < nr_pages; i++) {
360 struct page *p = page + i;
Christoph Lameterd85f3382007-05-06 14:49:39 -0700361 __SetPageTail(p);
Youquan Song58a84aa2011-12-08 14:34:18 -0800362 set_page_count(p, 0);
Christoph Lameterd85f3382007-05-06 14:49:39 -0700363 p->first_page = page;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364 }
365}
366
Andrea Arcangeli59ff4212011-01-13 15:46:38 -0800367/* update __split_huge_page_refcount if you change this function */
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800368static int destroy_compound_page(struct page *page, unsigned long order)
Andy Whitcroft18229df2008-11-06 12:53:27 -0800369{
370 int i;
371 int nr_pages = 1 << order;
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800372 int bad = 0;
Andy Whitcroft18229df2008-11-06 12:53:27 -0800373
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800374 if (unlikely(compound_order(page) != order) ||
375 unlikely(!PageHead(page))) {
Nick Piggin224abf92006-01-06 00:11:11 -0800376 bad_page(page);
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800377 bad++;
378 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379
Christoph Lameter6d777952007-05-06 14:49:40 -0700380 __ClearPageHead(page);
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800381
Andy Whitcroft18229df2008-11-06 12:53:27 -0800382 for (i = 1; i < nr_pages; i++) {
383 struct page *p = page + i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700384
Alexey Zaytseve713a212009-01-10 02:47:57 +0300385 if (unlikely(!PageTail(p) || (p->first_page != page))) {
Nick Piggin224abf92006-01-06 00:11:11 -0800386 bad_page(page);
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800387 bad++;
388 }
Christoph Lameterd85f3382007-05-06 14:49:39 -0700389 __ClearPageTail(p);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 }
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800391
392 return bad;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700393}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700394
Nick Piggin17cf4402006-03-22 00:08:41 -0800395static inline void prep_zero_page(struct page *page, int order, gfp_t gfp_flags)
396{
397 int i;
398
Andrew Morton6626c5d2006-03-22 00:08:42 -0800399 /*
400 * clear_highpage() will use KM_USER0, so it's a bug to use __GFP_ZERO
401 * and __GFP_HIGHMEM from hard or soft interrupt context.
402 */
Nick Piggin725d7042006-09-25 23:30:55 -0700403 VM_BUG_ON((gfp_flags & __GFP_HIGHMEM) && in_interrupt());
Nick Piggin17cf4402006-03-22 00:08:41 -0800404 for (i = 0; i < (1 << order); i++)
405 clear_highpage(page + i);
406}
407
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800408#ifdef CONFIG_DEBUG_PAGEALLOC
409unsigned int _debug_guardpage_minorder;
410
411static int __init debug_guardpage_minorder_setup(char *buf)
412{
413 unsigned long res;
414
415 if (kstrtoul(buf, 10, &res) < 0 || res > MAX_ORDER / 2) {
416 printk(KERN_ERR "Bad debug_guardpage_minorder value\n");
417 return 0;
418 }
419 _debug_guardpage_minorder = res;
420 printk(KERN_INFO "Setting debug_guardpage_minorder to %lu\n", res);
421 return 0;
422}
423__setup("debug_guardpage_minorder=", debug_guardpage_minorder_setup);
424
425static inline void set_page_guard_flag(struct page *page)
426{
427 __set_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);
428}
429
430static inline void clear_page_guard_flag(struct page *page)
431{
432 __clear_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);
433}
434#else
435static inline void set_page_guard_flag(struct page *page) { }
436static inline void clear_page_guard_flag(struct page *page) { }
437#endif
438
Andrew Morton6aa30012006-04-18 22:20:52 -0700439static inline void set_page_order(struct page *page, int order)
440{
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700441 set_page_private(page, order);
Nick Piggin676165a2006-04-10 11:21:48 +1000442 __SetPageBuddy(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700443}
444
445static inline void rmv_page_order(struct page *page)
446{
Nick Piggin676165a2006-04-10 11:21:48 +1000447 __ClearPageBuddy(page);
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700448 set_page_private(page, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449}
450
451/*
452 * Locate the struct page for both the matching buddy in our
453 * pair (buddy1) and the combined O(n+1) page they form (page).
454 *
455 * 1) Any buddy B1 will have an order O twin B2 which satisfies
456 * the following equation:
457 * B2 = B1 ^ (1 << O)
458 * For example, if the starting buddy (buddy2) is #8 its order
459 * 1 buddy is #10:
460 * B2 = 8 ^ (1 << 1) = 8 ^ 2 = 10
461 *
462 * 2) Any buddy B will have an order O+1 parent P which
463 * satisfies the following equation:
464 * P = B & ~(1 << O)
465 *
Andreas Mohrd6e05ed2006-06-26 18:35:02 +0200466 * Assumption: *_mem_map is contiguous at least up to MAX_ORDER
Linus Torvalds1da177e2005-04-16 15:20:36 -0700467 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700468static inline unsigned long
KyongHo Cho43506fa2011-01-13 15:47:24 -0800469__find_buddy_index(unsigned long page_idx, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470{
KyongHo Cho43506fa2011-01-13 15:47:24 -0800471 return page_idx ^ (1 << order);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700472}
473
474/*
475 * This function checks whether a page is free && is the buddy
476 * we can do coalesce a page and its buddy if
Nick Piggin13e74442006-01-06 00:10:58 -0800477 * (a) the buddy is not in a hole &&
Nick Piggin676165a2006-04-10 11:21:48 +1000478 * (b) the buddy is in the buddy system &&
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -0700479 * (c) a page and its buddy have the same order &&
480 * (d) a page and its buddy are in the same zone.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700481 *
Andrea Arcangeli5f24ce52011-01-13 15:47:00 -0800482 * For recording whether a page is in the buddy system, we set ->_mapcount -2.
483 * Setting, clearing, and testing _mapcount -2 is serialized by zone->lock.
Nick Piggin676165a2006-04-10 11:21:48 +1000484 *
485 * For recording page's order, we use page_private(page).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700486 */
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -0700487static inline int page_is_buddy(struct page *page, struct page *buddy,
488 int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700489{
Andy Whitcroft14e07292007-05-06 14:49:14 -0700490 if (!pfn_valid_within(page_to_pfn(buddy)))
Nick Piggin13e74442006-01-06 00:10:58 -0800491 return 0;
Nick Piggin13e74442006-01-06 00:10:58 -0800492
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -0700493 if (page_zone_id(page) != page_zone_id(buddy))
494 return 0;
495
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800496 if (page_is_guard(buddy) && page_order(buddy) == order) {
497 VM_BUG_ON(page_count(buddy) != 0);
498 return 1;
499 }
500
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -0700501 if (PageBuddy(buddy) && page_order(buddy) == order) {
Nick Piggina3af9c32009-06-16 15:32:10 -0700502 VM_BUG_ON(page_count(buddy) != 0);
Andrew Morton6aa30012006-04-18 22:20:52 -0700503 return 1;
Nick Piggin676165a2006-04-10 11:21:48 +1000504 }
Andrew Morton6aa30012006-04-18 22:20:52 -0700505 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700506}
507
508/*
509 * Freeing function for a buddy system allocator.
510 *
511 * The concept of a buddy system is to maintain direct-mapped table
512 * (containing bit values) for memory blocks of various "orders".
513 * The bottom level table contains the map for the smallest allocatable
514 * units of memory (here, pages), and each level above it describes
515 * pairs of units from the levels below, hence, "buddies".
516 * At a high level, all that happens here is marking the table entry
517 * at the bottom level available, and propagating the changes upward
518 * as necessary, plus some accounting needed to play nicely with other
519 * parts of the VM system.
520 * At each level, we keep a list of pages, which are heads of continuous
Andrea Arcangeli5f24ce52011-01-13 15:47:00 -0800521 * free pages of length of (1 << order) and marked with _mapcount -2. Page's
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700522 * order is recorded in page_private(page) field.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700523 * So when we are allocating or freeing one, we can derive the state of the
Michal Nazarewiczadc95fd2012-01-11 15:16:11 +0100524 * other. That is, if we allocate a small block, and both were
525 * free, the remainder of the region must be split into blocks.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700526 * If a block is freed, and its buddy is also free, then this
Michal Nazarewiczadc95fd2012-01-11 15:16:11 +0100527 * triggers coalescing into a block of larger size.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700528 *
529 * -- wli
530 */
531
Nick Piggin48db57f2006-01-08 01:00:42 -0800532static inline void __free_one_page(struct page *page,
Mel Gormaned0ae212009-06-16 15:32:07 -0700533 struct zone *zone, unsigned int order,
534 int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535{
536 unsigned long page_idx;
Corrado Zoccolo6dda9d52010-05-24 14:31:54 -0700537 unsigned long combined_idx;
KyongHo Cho43506fa2011-01-13 15:47:24 -0800538 unsigned long uninitialized_var(buddy_idx);
Shashank Mittal53103492012-06-19 19:45:35 -0700539 struct page *buddy = NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700540
Nick Piggin224abf92006-01-06 00:11:11 -0800541 if (unlikely(PageCompound(page)))
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800542 if (unlikely(destroy_compound_page(page, order)))
543 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544
Mel Gormaned0ae212009-06-16 15:32:07 -0700545 VM_BUG_ON(migratetype == -1);
546
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547 page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
548
Mel Gormanf2260e62009-06-16 15:32:13 -0700549 VM_BUG_ON(page_idx & ((1 << order) - 1));
Nick Piggin725d7042006-09-25 23:30:55 -0700550 VM_BUG_ON(bad_range(zone, page));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700551
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552 while (order < MAX_ORDER-1) {
KyongHo Cho43506fa2011-01-13 15:47:24 -0800553 buddy_idx = __find_buddy_index(page_idx, order);
554 buddy = page + (buddy_idx - page_idx);
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -0700555 if (!page_is_buddy(page, buddy, order))
Andy Whitcroft3c82d0c2008-07-23 21:27:11 -0700556 break;
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800557 /*
558 * Our buddy is free or it is CONFIG_DEBUG_PAGEALLOC guard page,
559 * merge with it and move up one order.
560 */
561 if (page_is_guard(buddy)) {
562 clear_page_guard_flag(buddy);
563 set_page_private(page, 0);
Larry Bassel6d6e2c92012-12-14 14:21:05 -0800564 __mod_zone_freepage_state(zone, 1 << order,
565 migratetype);
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800566 } else {
567 list_del(&buddy->lru);
568 zone->free_area[order].nr_free--;
569 rmv_page_order(buddy);
570 }
KyongHo Cho43506fa2011-01-13 15:47:24 -0800571 combined_idx = buddy_idx & page_idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700572 page = page + (combined_idx - page_idx);
573 page_idx = combined_idx;
574 order++;
575 }
576 set_page_order(page, order);
Corrado Zoccolo6dda9d52010-05-24 14:31:54 -0700577
578 /*
579 * If this is not the largest possible page, check if the buddy
580 * of the next-highest order is free. If it is, it's possible
581 * that pages are being freed that will coalesce soon. In case,
582 * that is happening, add the free page to the tail of the list
583 * so it's less likely to be used soon and more likely to be merged
584 * as a higher order page
585 */
Mel Gormanb7f50cf2010-10-26 14:21:11 -0700586 if ((order < MAX_ORDER-2) && pfn_valid_within(page_to_pfn(buddy))) {
Corrado Zoccolo6dda9d52010-05-24 14:31:54 -0700587 struct page *higher_page, *higher_buddy;
KyongHo Cho43506fa2011-01-13 15:47:24 -0800588 combined_idx = buddy_idx & page_idx;
589 higher_page = page + (combined_idx - page_idx);
590 buddy_idx = __find_buddy_index(combined_idx, order + 1);
591 higher_buddy = page + (buddy_idx - combined_idx);
Corrado Zoccolo6dda9d52010-05-24 14:31:54 -0700592 if (page_is_buddy(higher_page, higher_buddy, order + 1)) {
593 list_add_tail(&page->lru,
594 &zone->free_area[order].free_list[migratetype]);
595 goto out;
596 }
597 }
598
599 list_add(&page->lru, &zone->free_area[order].free_list[migratetype]);
600out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700601 zone->free_area[order].nr_free++;
602}
603
KOSAKI Motohiro092cead2009-06-16 15:32:17 -0700604/*
605 * free_page_mlock() -- clean up attempts to free and mlocked() page.
606 * Page should not be on lru, so no need to fix that up.
607 * free_pages_check() will verify...
608 */
609static inline void free_page_mlock(struct page *page)
610{
KOSAKI Motohiro092cead2009-06-16 15:32:17 -0700611 __dec_zone_page_state(page, NR_MLOCK);
612 __count_vm_event(UNEVICTABLE_MLOCKFREED);
613}
KOSAKI Motohiro092cead2009-06-16 15:32:17 -0700614
Nick Piggin224abf92006-01-06 00:11:11 -0800615static inline int free_pages_check(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616{
Nick Piggin92be2e32006-01-06 00:10:57 -0800617 if (unlikely(page_mapcount(page) |
618 (page->mapping != NULL) |
Nick Piggina3af9c32009-06-16 15:32:10 -0700619 (atomic_read(&page->_count) != 0) |
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -0700620 (page->flags & PAGE_FLAGS_CHECK_AT_FREE) |
621 (mem_cgroup_bad_page_check(page)))) {
Nick Piggin224abf92006-01-06 00:11:11 -0800622 bad_page(page);
Hugh Dickins79f4b7b2009-01-06 14:40:05 -0800623 return 1;
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800624 }
Hugh Dickins79f4b7b2009-01-06 14:40:05 -0800625 if (page->flags & PAGE_FLAGS_CHECK_AT_PREP)
626 page->flags &= ~PAGE_FLAGS_CHECK_AT_PREP;
627 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700628}
629
630/*
Mel Gorman5f8dcc22009-09-21 17:03:19 -0700631 * Frees a number of pages from the PCP lists
Linus Torvalds1da177e2005-04-16 15:20:36 -0700632 * Assumes all pages on list are in same zone, and of same order.
Renaud Lienhart207f36e2005-09-10 00:26:59 -0700633 * count is the number of pages to free.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634 *
635 * If the zone was previously in an "all pages pinned" state then look to
636 * see if this freeing clears that state.
637 *
638 * And clear the zone's pages_scanned counter, to hold off the "all pages are
639 * pinned" detection logic.
640 */
Mel Gorman5f8dcc22009-09-21 17:03:19 -0700641static void free_pcppages_bulk(struct zone *zone, int count,
642 struct per_cpu_pages *pcp)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643{
Mel Gorman5f8dcc22009-09-21 17:03:19 -0700644 int migratetype = 0;
Mel Gormana6f9edd2009-09-21 17:03:20 -0700645 int batch_free = 0;
Mel Gorman72853e22010-09-09 16:38:16 -0700646 int to_free = count;
Larry Bassel6d6e2c92012-12-14 14:21:05 -0800647 int mt = 0;
Mel Gorman5f8dcc22009-09-21 17:03:19 -0700648
Nick Pigginc54ad302006-01-06 00:10:56 -0800649 spin_lock(&zone->lock);
KOSAKI Motohiro93e4a892010-03-05 13:41:55 -0800650 zone->all_unreclaimable = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700651 zone->pages_scanned = 0;
Mel Gormanf2260e62009-06-16 15:32:13 -0700652
Mel Gorman72853e22010-09-09 16:38:16 -0700653 while (to_free) {
Nick Piggin48db57f2006-01-08 01:00:42 -0800654 struct page *page;
Mel Gorman5f8dcc22009-09-21 17:03:19 -0700655 struct list_head *list;
Nick Piggin48db57f2006-01-08 01:00:42 -0800656
Mel Gorman5f8dcc22009-09-21 17:03:19 -0700657 /*
Mel Gormana6f9edd2009-09-21 17:03:20 -0700658 * Remove pages from lists in a round-robin fashion. A
659 * batch_free count is maintained that is incremented when an
660 * empty list is encountered. This is so more pages are freed
661 * off fuller lists instead of spinning excessively around empty
662 * lists
Mel Gorman5f8dcc22009-09-21 17:03:19 -0700663 */
664 do {
Mel Gormana6f9edd2009-09-21 17:03:20 -0700665 batch_free++;
Mel Gorman5f8dcc22009-09-21 17:03:19 -0700666 if (++migratetype == MIGRATE_PCPTYPES)
667 migratetype = 0;
668 list = &pcp->lists[migratetype];
669 } while (list_empty(list));
670
Namhyung Kim1d168712011-03-22 16:32:45 -0700671 /* This is the only non-empty list. Free them all. */
672 if (batch_free == MIGRATE_PCPTYPES)
673 batch_free = to_free;
674
Mel Gormana6f9edd2009-09-21 17:03:20 -0700675 do {
676 page = list_entry(list->prev, struct page, lru);
Larry Bassel6d6e2c92012-12-14 14:21:05 -0800677 mt = get_pageblock_migratetype(page);
Mel Gormana6f9edd2009-09-21 17:03:20 -0700678 /* must delete as __free_one_page list manipulates */
679 list_del(&page->lru);
Hugh Dickinsa7016232010-01-29 17:46:34 +0000680 /* MIGRATE_MOVABLE list may include MIGRATE_RESERVEs */
681 __free_one_page(page, zone, 0, page_private(page));
682 trace_mm_page_pcpu_drain(page, 0, page_private(page));
Larry Bassel6d6e2c92012-12-14 14:21:05 -0800683 if (is_migrate_cma(mt))
684 __mod_zone_page_state(zone,
685 NR_FREE_CMA_PAGES, 1);
Mel Gorman72853e22010-09-09 16:38:16 -0700686 } while (--to_free && --batch_free && !list_empty(list));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687 }
Mel Gorman72853e22010-09-09 16:38:16 -0700688 __mod_zone_page_state(zone, NR_FREE_PAGES, count);
Nick Pigginc54ad302006-01-06 00:10:56 -0800689 spin_unlock(&zone->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700690}
691
Mel Gormaned0ae212009-06-16 15:32:07 -0700692static void free_one_page(struct zone *zone, struct page *page, int order,
693 int migratetype)
Nick Piggin48db57f2006-01-08 01:00:42 -0800694{
Christoph Lameter006d22d2006-09-25 23:31:48 -0700695 spin_lock(&zone->lock);
KOSAKI Motohiro93e4a892010-03-05 13:41:55 -0800696 zone->all_unreclaimable = 0;
Christoph Lameter006d22d2006-09-25 23:31:48 -0700697 zone->pages_scanned = 0;
Mel Gormanf2260e62009-06-16 15:32:13 -0700698
Mel Gormaned0ae212009-06-16 15:32:07 -0700699 __free_one_page(page, zone, order, migratetype);
Larry Bassel6d6e2c92012-12-14 14:21:05 -0800700 if (unlikely(migratetype != MIGRATE_ISOLATE))
701 __mod_zone_freepage_state(zone, 1 << order, migratetype);
Christoph Lameter006d22d2006-09-25 23:31:48 -0700702 spin_unlock(&zone->lock);
Nick Piggin48db57f2006-01-08 01:00:42 -0800703}
704
KOSAKI Motohiroec95f532010-05-24 14:32:38 -0700705static bool free_pages_prepare(struct page *page, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707 int i;
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800708 int bad = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700709
Konstantin Khlebnikovb413d482012-01-10 15:07:09 -0800710 trace_mm_page_free(page, order);
Vegard Nossumb1eeab62008-11-25 16:55:53 +0100711 kmemcheck_free_shadow(page, order);
712
Andrea Arcangeli8dd60a32011-01-13 15:46:34 -0800713 if (PageAnon(page))
714 page->mapping = NULL;
715 for (i = 0; i < (1 << order); i++)
716 bad += free_pages_check(page + i);
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800717 if (bad)
KOSAKI Motohiroec95f532010-05-24 14:32:38 -0700718 return false;
Hugh Dickins689bceb2005-11-21 21:32:20 -0800719
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -0700720 if (!PageHighMem(page)) {
Nick Piggin9858db52006-10-11 01:21:30 -0700721 debug_check_no_locks_freed(page_address(page),PAGE_SIZE<<order);
Thomas Gleixner3ac7fe52008-04-30 00:55:01 -0700722 debug_check_no_obj_freed(page_address(page),
723 PAGE_SIZE << order);
724 }
Nick Piggindafb1362006-10-11 01:21:30 -0700725 arch_free_page(page, order);
Nick Piggin48db57f2006-01-08 01:00:42 -0800726 kernel_map_pages(page, 1 << order, 0);
Nick Piggindafb1362006-10-11 01:21:30 -0700727
KOSAKI Motohiroec95f532010-05-24 14:32:38 -0700728 return true;
729}
730
731static void __free_pages_ok(struct page *page, unsigned int order)
732{
733 unsigned long flags;
734 int wasMlocked = __TestClearPageMlocked(page);
735
736 if (!free_pages_prepare(page, order))
737 return;
738
Nick Pigginc54ad302006-01-06 00:10:56 -0800739 local_irq_save(flags);
Johannes Weinerc2773312009-06-19 19:30:56 +0200740 if (unlikely(wasMlocked))
Mel Gormanda456f12009-06-16 15:32:08 -0700741 free_page_mlock(page);
Christoph Lameterf8891e52006-06-30 01:55:45 -0700742 __count_vm_events(PGFREE, 1 << order);
Mel Gormaned0ae212009-06-16 15:32:07 -0700743 free_one_page(page_zone(page), page, order,
744 get_pageblock_migratetype(page));
Nick Pigginc54ad302006-01-06 00:10:56 -0800745 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700746}
747
Laura Abbottf8382d82013-06-28 12:52:17 -0700748void __free_pages_bootmem(struct page *page, unsigned int order)
David Howellsa226f6c2006-01-06 00:11:08 -0800749{
Johannes Weinerc3993072012-01-10 15:08:10 -0800750 unsigned int nr_pages = 1 << order;
751 unsigned int loop;
David Howellsa226f6c2006-01-06 00:11:08 -0800752
Johannes Weinerc3993072012-01-10 15:08:10 -0800753 prefetchw(page);
754 for (loop = 0; loop < nr_pages; loop++) {
755 struct page *p = &page[loop];
David Howellsa226f6c2006-01-06 00:11:08 -0800756
Johannes Weinerc3993072012-01-10 15:08:10 -0800757 if (loop + 1 < nr_pages)
758 prefetchw(p + 1);
759 __ClearPageReserved(p);
760 set_page_count(p, 0);
David Howellsa226f6c2006-01-06 00:11:08 -0800761 }
Johannes Weinerc3993072012-01-10 15:08:10 -0800762
763 set_page_refcounted(page);
764 __free_pages(page, order);
David Howellsa226f6c2006-01-06 00:11:08 -0800765}
766
Michal Nazarewiczd4158d22011-12-29 13:09:50 +0100767#ifdef CONFIG_CMA
Laura Abbott364dcd42012-11-27 10:17:24 -0800768bool is_cma_pageblock(struct page *page)
769{
770 return get_pageblock_migratetype(page) == MIGRATE_CMA;
771}
772
Michal Nazarewiczd4158d22011-12-29 13:09:50 +0100773/* Free whole pageblock and set it's migration type to MIGRATE_CMA. */
774void __init init_cma_reserved_pageblock(struct page *page)
775{
776 unsigned i = pageblock_nr_pages;
777 struct page *p = page;
778
779 do {
780 __ClearPageReserved(p);
781 set_page_count(p, 0);
782 } while (++p, --i);
783
784 set_page_refcounted(page);
785 set_pageblock_migratetype(page, MIGRATE_CMA);
786 __free_pages(page, pageblock_order);
787 totalram_pages += pageblock_nr_pages;
Marek Szyprowski178a68e2013-02-12 13:46:24 -0800788#ifdef CONFIG_HIGHMEM
789 if (PageHighMem(page))
790 totalhigh_pages += pageblock_nr_pages;
791#endif
Michal Nazarewiczd4158d22011-12-29 13:09:50 +0100792}
793#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700794
795/*
796 * The order of subdivision here is critical for the IO subsystem.
797 * Please do not alter this order without good reasons and regression
798 * testing. Specifically, as large blocks of memory are subdivided,
799 * the order in which smaller blocks are delivered depends on the order
800 * they're subdivided in this function. This is the primary factor
801 * influencing the order in which pages are delivered to the IO
802 * subsystem according to empirical testing, and this is also justified
803 * by considering the behavior of a buddy system containing a single
804 * large block of memory acted on by a series of small allocations.
805 * This behavior is a critical factor in sglist merging's success.
806 *
807 * -- wli
808 */
Nick Piggin085cc7d2006-01-06 00:11:01 -0800809static inline void expand(struct zone *zone, struct page *page,
Mel Gormanb2a0ac82007-10-16 01:25:48 -0700810 int low, int high, struct free_area *area,
811 int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700812{
813 unsigned long size = 1 << high;
814
815 while (high > low) {
816 area--;
817 high--;
818 size >>= 1;
Nick Piggin725d7042006-09-25 23:30:55 -0700819 VM_BUG_ON(bad_range(zone, &page[size]));
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800820
821#ifdef CONFIG_DEBUG_PAGEALLOC
822 if (high < debug_guardpage_minorder()) {
823 /*
824 * Mark as guard pages (or page), that will allow to
825 * merge back to allocator when buddy will be freed.
826 * Corresponding page table entries will not be touched,
827 * pages will stay not present in virtual address space
828 */
829 INIT_LIST_HEAD(&page[size].lru);
830 set_page_guard_flag(&page[size]);
831 set_page_private(&page[size], high);
832 /* Guard pages are not available for any usage */
Larry Bassel6d6e2c92012-12-14 14:21:05 -0800833 __mod_zone_freepage_state(zone, -(1 << high),
834 migratetype);
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -0800835 continue;
836 }
837#endif
Mel Gormanb2a0ac82007-10-16 01:25:48 -0700838 list_add(&page[size].lru, &area->free_list[migratetype]);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839 area->nr_free++;
840 set_page_order(&page[size], high);
841 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842}
843
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844/*
845 * This page is about to be returned from the page allocator
846 */
Wu Fengguang2a7684a2009-09-16 11:50:12 +0200847static inline int check_new_page(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700848{
Nick Piggin92be2e32006-01-06 00:10:57 -0800849 if (unlikely(page_mapcount(page) |
850 (page->mapping != NULL) |
Nick Piggina3af9c32009-06-16 15:32:10 -0700851 (atomic_read(&page->_count) != 0) |
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -0700852 (page->flags & PAGE_FLAGS_CHECK_AT_PREP) |
853 (mem_cgroup_bad_page_check(page)))) {
Nick Piggin224abf92006-01-06 00:11:11 -0800854 bad_page(page);
Hugh Dickins689bceb2005-11-21 21:32:20 -0800855 return 1;
Hugh Dickins8cc3b392009-01-06 14:40:06 -0800856 }
Wu Fengguang2a7684a2009-09-16 11:50:12 +0200857 return 0;
858}
859
860static int prep_new_page(struct page *page, int order, gfp_t gfp_flags)
861{
862 int i;
863
864 for (i = 0; i < (1 << order); i++) {
865 struct page *p = page + i;
866 if (unlikely(check_new_page(p)))
867 return 1;
868 }
Hugh Dickins689bceb2005-11-21 21:32:20 -0800869
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700870 set_page_private(page, 0);
Nick Piggin7835e982006-03-22 00:08:40 -0800871 set_page_refcounted(page);
Nick Piggincc102502006-12-06 20:32:00 -0800872
873 arch_alloc_page(page, order);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700874 kernel_map_pages(page, 1 << order, 1);
Nick Piggin17cf4402006-03-22 00:08:41 -0800875
876 if (gfp_flags & __GFP_ZERO)
877 prep_zero_page(page, order, gfp_flags);
878
879 if (order && (gfp_flags & __GFP_COMP))
880 prep_compound_page(page, order);
881
Hugh Dickins689bceb2005-11-21 21:32:20 -0800882 return 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700883}
884
Mel Gorman56fd56b2007-10-16 01:25:58 -0700885/*
886 * Go through the free lists for the given migratetype and remove
887 * the smallest available page from the freelists
888 */
Mel Gorman728ec982009-06-16 15:32:04 -0700889static inline
890struct page *__rmqueue_smallest(struct zone *zone, unsigned int order,
Mel Gorman56fd56b2007-10-16 01:25:58 -0700891 int migratetype)
892{
893 unsigned int current_order;
894 struct free_area * area;
895 struct page *page;
896
897 /* Find a page of the appropriate size in the preferred list */
898 for (current_order = order; current_order < MAX_ORDER; ++current_order) {
899 area = &(zone->free_area[current_order]);
900 if (list_empty(&area->free_list[migratetype]))
901 continue;
902
903 page = list_entry(area->free_list[migratetype].next,
904 struct page, lru);
905 list_del(&page->lru);
906 rmv_page_order(page);
907 area->nr_free--;
Mel Gorman56fd56b2007-10-16 01:25:58 -0700908 expand(zone, page, order, current_order, area, migratetype);
909 return page;
910 }
911
912 return NULL;
913}
914
915
Mel Gormanb2a0ac82007-10-16 01:25:48 -0700916/*
917 * This array describes the order lists are fallen back to when
918 * the free lists for the desirable migrate type are depleted
919 */
Laura Abbott434f55a2013-02-18 07:17:06 -0800920static int fallbacks[MIGRATE_TYPES][4] = {
Michal Nazarewiczd4158d22011-12-29 13:09:50 +0100921 [MIGRATE_UNMOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
922 [MIGRATE_RECLAIMABLE] = { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RESERVE },
Michal Nazarewiczb5662d62012-11-20 16:37:50 +0100923#ifdef CONFIG_CMA
Laura Abbott434f55a2013-02-18 07:17:06 -0800924 [MIGRATE_MOVABLE] = { MIGRATE_CMA, MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
Michal Nazarewiczb5662d62012-11-20 16:37:50 +0100925 [MIGRATE_CMA] = { MIGRATE_RESERVE }, /* Never used */
Laura Abbott434f55a2013-02-18 07:17:06 -0800926#else
927 [MIGRATE_MOVABLE] = { MIGRATE_RECLAIMABLE, MIGRATE_UNMOVABLE, MIGRATE_RESERVE },
Michal Nazarewiczd4158d22011-12-29 13:09:50 +0100928#endif
Michal Nazarewiczc6114bf2012-01-11 15:31:33 +0100929 [MIGRATE_RESERVE] = { MIGRATE_RESERVE }, /* Never used */
930 [MIGRATE_ISOLATE] = { MIGRATE_RESERVE }, /* Never used */
Mel Gormanb2a0ac82007-10-16 01:25:48 -0700931};
932
Liam Mark3f1e5512013-03-27 12:34:51 -0700933int *get_migratetype_fallbacks(int mtype)
934{
935 return fallbacks[mtype];
936}
937
Mel Gormanc361be52007-10-16 01:25:51 -0700938/*
939 * Move the free pages in a range to the free lists of the requested type.
Mel Gormand9c23402007-10-16 01:26:01 -0700940 * Note that start_page and end_pages are not aligned on a pageblock
Mel Gormanc361be52007-10-16 01:25:51 -0700941 * boundary. If alignment is required, use move_freepages_block()
942 */
Adrian Bunkb69a7282008-07-23 21:28:12 -0700943static int move_freepages(struct zone *zone,
944 struct page *start_page, struct page *end_page,
945 int migratetype)
Mel Gormanc361be52007-10-16 01:25:51 -0700946{
947 struct page *page;
948 unsigned long order;
Mel Gormand1003132007-10-16 01:26:00 -0700949 int pages_moved = 0;
Mel Gormanc361be52007-10-16 01:25:51 -0700950
951#ifndef CONFIG_HOLES_IN_ZONE
952 /*
953 * page_zone is not safe to call in this context when
954 * CONFIG_HOLES_IN_ZONE is set. This bug check is probably redundant
955 * anyway as we check zone boundaries in move_freepages_block().
956 * Remove at a later date when no bug reports exist related to
Mel Gormanac0e5b72007-10-16 01:25:58 -0700957 * grouping pages by mobility
Mel Gormanc361be52007-10-16 01:25:51 -0700958 */
959 BUG_ON(page_zone(start_page) != page_zone(end_page));
960#endif
961
962 for (page = start_page; page <= end_page;) {
Adam Litke344c7902008-09-02 14:35:38 -0700963 /* Make sure we are not inadvertently changing nodes */
964 VM_BUG_ON(page_to_nid(page) != zone_to_nid(zone));
965
Mel Gormanc361be52007-10-16 01:25:51 -0700966 if (!pfn_valid_within(page_to_pfn(page))) {
967 page++;
968 continue;
969 }
970
971 if (!PageBuddy(page)) {
972 page++;
973 continue;
974 }
975
976 order = page_order(page);
Kirill A. Shutemov84be48d2011-03-22 16:33:41 -0700977 list_move(&page->lru,
978 &zone->free_area[order].free_list[migratetype]);
Mel Gormanc361be52007-10-16 01:25:51 -0700979 page += 1 << order;
Mel Gormand1003132007-10-16 01:26:00 -0700980 pages_moved += 1 << order;
Mel Gormanc361be52007-10-16 01:25:51 -0700981 }
982
Mel Gormand1003132007-10-16 01:26:00 -0700983 return pages_moved;
Mel Gormanc361be52007-10-16 01:25:51 -0700984}
985
Adrian Bunkb69a7282008-07-23 21:28:12 -0700986static int move_freepages_block(struct zone *zone, struct page *page,
987 int migratetype)
Mel Gormanc361be52007-10-16 01:25:51 -0700988{
989 unsigned long start_pfn, end_pfn;
990 struct page *start_page, *end_page;
991
992 start_pfn = page_to_pfn(page);
Mel Gormand9c23402007-10-16 01:26:01 -0700993 start_pfn = start_pfn & ~(pageblock_nr_pages-1);
Mel Gormanc361be52007-10-16 01:25:51 -0700994 start_page = pfn_to_page(start_pfn);
Mel Gormand9c23402007-10-16 01:26:01 -0700995 end_page = start_page + pageblock_nr_pages - 1;
996 end_pfn = start_pfn + pageblock_nr_pages - 1;
Mel Gormanc361be52007-10-16 01:25:51 -0700997
998 /* Do not cross zone boundaries */
999 if (start_pfn < zone->zone_start_pfn)
1000 start_page = page;
1001 if (end_pfn >= zone->zone_start_pfn + zone->spanned_pages)
1002 return 0;
1003
1004 return move_freepages(zone, start_page, end_page, migratetype);
1005}
1006
Mel Gorman2f66a682009-09-21 17:02:31 -07001007static void change_pageblock_range(struct page *pageblock_page,
1008 int start_order, int migratetype)
1009{
1010 int nr_pageblocks = 1 << (start_order - pageblock_order);
1011
1012 while (nr_pageblocks--) {
1013 set_pageblock_migratetype(pageblock_page, migratetype);
1014 pageblock_page += pageblock_nr_pages;
1015 }
1016}
1017
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001018/* Remove an element from the buddy allocator from the fallback list */
Mel Gorman0ac3a402009-06-16 15:32:06 -07001019static inline struct page *
1020__rmqueue_fallback(struct zone *zone, int order, int start_migratetype)
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001021{
1022 struct free_area * area;
1023 int current_order;
1024 struct page *page;
1025 int migratetype, i;
1026
1027 /* Find the largest possible block of pages in the other list */
1028 for (current_order = MAX_ORDER-1; current_order >= order;
1029 --current_order) {
Michal Nazarewiczc6114bf2012-01-11 15:31:33 +01001030 for (i = 0;; i++) {
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001031 migratetype = fallbacks[start_migratetype][i];
1032
Mel Gorman56fd56b2007-10-16 01:25:58 -07001033 /* MIGRATE_RESERVE handled later if necessary */
1034 if (migratetype == MIGRATE_RESERVE)
Michal Nazarewiczc6114bf2012-01-11 15:31:33 +01001035 break;
Mel Gormane0104872007-10-16 01:25:53 -07001036
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001037 area = &(zone->free_area[current_order]);
1038 if (list_empty(&area->free_list[migratetype]))
1039 continue;
1040
1041 page = list_entry(area->free_list[migratetype].next,
1042 struct page, lru);
1043 area->nr_free--;
1044
1045 /*
Mel Gormanc361be52007-10-16 01:25:51 -07001046 * If breaking a large block of pages, move all free
Mel Gorman46dafbc2007-10-16 01:25:55 -07001047 * pages to the preferred allocation list. If falling
1048 * back for a reclaimable kernel allocation, be more
Lucas De Marchi25985ed2011-03-30 22:57:33 -03001049 * aggressive about taking ownership of free pages
Laura Abbott434f55a2013-02-18 07:17:06 -08001050 *
1051 * On the other hand, never change migration
1052 * type of MIGRATE_CMA pageblocks nor move CMA
1053 * pages on different free lists. We don't
1054 * want unmovable pages to be allocated from
1055 * MIGRATE_CMA areas.
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001056 */
Laura Abbott434f55a2013-02-18 07:17:06 -08001057 if (!is_migrate_cma(migratetype) &&
1058 (unlikely(current_order >= pageblock_order / 2) ||
1059 start_migratetype == MIGRATE_RECLAIMABLE ||
1060 page_group_by_mobility_disabled)) {
Michal Nazarewiczd4158d22011-12-29 13:09:50 +01001061 int pages;
Mel Gorman46dafbc2007-10-16 01:25:55 -07001062 pages = move_freepages_block(zone, page,
1063 start_migratetype);
1064
1065 /* Claim the whole block if over half of it is free */
Mel Gormandd5d2412009-09-05 11:17:11 -07001066 if (pages >= (1 << (pageblock_order-1)) ||
1067 page_group_by_mobility_disabled)
Mel Gorman46dafbc2007-10-16 01:25:55 -07001068 set_pageblock_migratetype(page,
1069 start_migratetype);
1070
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001071 migratetype = start_migratetype;
Mel Gormanc361be52007-10-16 01:25:51 -07001072 }
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001073
1074 /* Remove the page from the freelists */
1075 list_del(&page->lru);
1076 rmv_page_order(page);
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001077
Mel Gorman2f66a682009-09-21 17:02:31 -07001078 /* Take ownership for orders >= pageblock_order */
Laura Abbott434f55a2013-02-18 07:17:06 -08001079 if (current_order >= pageblock_order &&
1080 !is_migrate_cma(migratetype))
Mel Gorman2f66a682009-09-21 17:02:31 -07001081 change_pageblock_range(page, current_order,
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001082 start_migratetype);
1083
Michal Nazarewiczd4158d22011-12-29 13:09:50 +01001084 expand(zone, page, order, current_order, area,
Laura Abbott434f55a2013-02-18 07:17:06 -08001085 is_migrate_cma(migratetype)
1086 ? migratetype : start_migratetype);
Mel Gormane0fff1b2009-09-21 17:02:42 -07001087
1088 trace_mm_page_alloc_extfrag(page, order, current_order,
1089 start_migratetype, migratetype);
1090
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001091 return page;
1092 }
1093 }
1094
Mel Gorman728ec982009-06-16 15:32:04 -07001095 return NULL;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001096}
1097
Mel Gorman56fd56b2007-10-16 01:25:58 -07001098/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099 * Do the hard work of removing an element from the buddy allocator.
1100 * Call me with the zone->lock already held.
1101 */
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001102static struct page *__rmqueue(struct zone *zone, unsigned int order,
1103 int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001104{
Linus Torvalds1da177e2005-04-16 15:20:36 -07001105 struct page *page;
1106
Laura Abbott434f55a2013-02-18 07:17:06 -08001107retry_reserve:
1108 page = __rmqueue_smallest(zone, order, migratetype);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001109
Laura Abbott434f55a2013-02-18 07:17:06 -08001110 if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
Michal Nazarewiczb5662d62012-11-20 16:37:50 +01001111 page = __rmqueue_fallback(zone, order, migratetype);
Michal Nazarewiczb5662d62012-11-20 16:37:50 +01001112
Laura Abbott434f55a2013-02-18 07:17:06 -08001113 /*
1114 * Use MIGRATE_RESERVE rather than fail an allocation. goto
1115 * is used because __rmqueue_smallest is an inline function
1116 * and we want just one call site
1117 */
1118 if (!page) {
1119 migratetype = MIGRATE_RESERVE;
1120 goto retry_reserve;
1121 }
Mel Gorman728ec982009-06-16 15:32:04 -07001122 }
1123
Mel Gorman0d3d0622009-09-21 17:02:44 -07001124 trace_mm_page_alloc_zone_locked(page, order, migratetype);
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001125 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126}
1127
Heesub Shin771aaa62013-01-07 11:10:13 +09001128static struct page *__rmqueue_cma(struct zone *zone, unsigned int order,
1129 int migratetype)
1130{
1131 struct page *page = 0;
1132#ifdef CONFIG_CMA
1133 if (migratetype == MIGRATE_MOVABLE && !zone->cma_alloc)
1134 page = __rmqueue_smallest(zone, order, MIGRATE_CMA);
Laura Abbotte2958b82013-04-05 12:39:18 -07001135 if (!page)
Heesub Shin771aaa62013-01-07 11:10:13 +09001136#endif
1137retry_reserve :
1138 page = __rmqueue_smallest(zone, order, migratetype);
1139
1140
1141 if (unlikely(!page) && migratetype != MIGRATE_RESERVE) {
1142 page = __rmqueue_fallback(zone, order, migratetype);
1143
1144 /*
1145 * Use MIGRATE_RESERVE rather than fail an allocation. goto
1146 * is used because __rmqueue_smallest is an inline function
1147 * and we want just one call site
1148 */
1149 if (!page) {
1150 migratetype = MIGRATE_RESERVE;
1151 goto retry_reserve;
1152 }
1153 }
1154
1155 trace_mm_page_alloc_zone_locked(page, order, migratetype);
1156 return page;
1157}
1158
Michal Nazarewiczadc95fd2012-01-11 15:16:11 +01001159/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001160 * Obtain a specified number of elements from the buddy allocator, all under
1161 * a single hold of the lock, for efficiency. Add them to the supplied list.
1162 * Returns the number of new pages which were placed at *list.
1163 */
Michal Nazarewiczadc95fd2012-01-11 15:16:11 +01001164static int rmqueue_bulk(struct zone *zone, unsigned int order,
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001165 unsigned long count, struct list_head *list,
Heesub Shin771aaa62013-01-07 11:10:13 +09001166 int migratetype, int cold, int cma)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001167{
Michal Nazarewiczd4158d22011-12-29 13:09:50 +01001168 int mt = migratetype, i;
Michal Nazarewiczadc95fd2012-01-11 15:16:11 +01001169
Nick Pigginc54ad302006-01-06 00:10:56 -08001170 spin_lock(&zone->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001171 for (i = 0; i < count; ++i) {
Heesub Shin771aaa62013-01-07 11:10:13 +09001172 struct page *page;
1173 if (cma)
1174 page = __rmqueue_cma(zone, order, migratetype);
1175 else
1176 page = __rmqueue(zone, order, migratetype);
Nick Piggin085cc7d2006-01-06 00:11:01 -08001177 if (unlikely(page == NULL))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178 break;
Mel Gorman81eabcb2007-12-17 16:20:05 -08001179
1180 /*
1181 * Split buddy pages returned by expand() are received here
1182 * in physical page order. The page is added to the callers and
1183 * list and the list head then moves forward. From the callers
1184 * perspective, the linked list is ordered by page number in
1185 * some conditions. This is useful for IO devices that can
1186 * merge IO requests if the physical pages are ordered
1187 * properly.
1188 */
Mel Gormane084b2d2009-07-29 15:02:04 -07001189 if (likely(cold == 0))
1190 list_add(&page->lru, list);
1191 else
1192 list_add_tail(&page->lru, list);
Michal Nazarewiczd4158d22011-12-29 13:09:50 +01001193 if (IS_ENABLED(CONFIG_CMA)) {
1194 mt = get_pageblock_migratetype(page);
1195 if (!is_migrate_cma(mt) && mt != MIGRATE_ISOLATE)
1196 mt = migratetype;
1197 }
1198 set_page_private(page, mt);
Mel Gorman81eabcb2007-12-17 16:20:05 -08001199 list = &page->lru;
Larry Bassel6d6e2c92012-12-14 14:21:05 -08001200 if (is_migrate_cma(mt))
1201 __mod_zone_page_state(zone, NR_FREE_CMA_PAGES,
1202 -(1 << order));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001203 }
Mel Gormanf2260e62009-06-16 15:32:13 -07001204 __mod_zone_page_state(zone, NR_FREE_PAGES, -(i << order));
Nick Pigginc54ad302006-01-06 00:10:56 -08001205 spin_unlock(&zone->lock);
Nick Piggin085cc7d2006-01-06 00:11:01 -08001206 return i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001207}
1208
Christoph Lameter4ae7c032005-06-21 17:14:57 -07001209#ifdef CONFIG_NUMA
Christoph Lameter8fce4d82006-03-09 17:33:54 -08001210/*
Christoph Lameter4037d452007-05-09 02:35:14 -07001211 * Called from the vmstat counter updater to drain pagesets of this
1212 * currently executing processor on remote nodes after they have
1213 * expired.
1214 *
Christoph Lameter879336c2006-03-22 00:09:08 -08001215 * Note that this function must be called with the thread pinned to
1216 * a single processor.
Christoph Lameter8fce4d82006-03-09 17:33:54 -08001217 */
Christoph Lameter4037d452007-05-09 02:35:14 -07001218void drain_zone_pages(struct zone *zone, struct per_cpu_pages *pcp)
Christoph Lameter4ae7c032005-06-21 17:14:57 -07001219{
Christoph Lameter4ae7c032005-06-21 17:14:57 -07001220 unsigned long flags;
Christoph Lameter4037d452007-05-09 02:35:14 -07001221 int to_drain;
Christoph Lameter4ae7c032005-06-21 17:14:57 -07001222
Christoph Lameter4037d452007-05-09 02:35:14 -07001223 local_irq_save(flags);
1224 if (pcp->count >= pcp->batch)
1225 to_drain = pcp->batch;
1226 else
1227 to_drain = pcp->count;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001228 free_pcppages_bulk(zone, to_drain, pcp);
Christoph Lameter4037d452007-05-09 02:35:14 -07001229 pcp->count -= to_drain;
1230 local_irq_restore(flags);
Christoph Lameter4ae7c032005-06-21 17:14:57 -07001231}
1232#endif
1233
Christoph Lameter9f8f2172008-02-04 22:29:11 -08001234/*
1235 * Drain pages of the indicated processor.
1236 *
1237 * The processor must either be the current processor and the
1238 * thread pinned to the current processor or a processor that
1239 * is not online.
1240 */
1241static void drain_pages(unsigned int cpu)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001242{
Nick Pigginc54ad302006-01-06 00:10:56 -08001243 unsigned long flags;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001244 struct zone *zone;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001245
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07001246 for_each_populated_zone(zone) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001247 struct per_cpu_pageset *pset;
Christoph Lameter3dfa5722008-02-04 22:29:19 -08001248 struct per_cpu_pages *pcp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001249
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09001250 local_irq_save(flags);
1251 pset = per_cpu_ptr(zone->pageset, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001252
Christoph Lameter3dfa5722008-02-04 22:29:19 -08001253 pcp = &pset->pcp;
David Rientjes2ff754f2011-01-25 15:07:23 -08001254 if (pcp->count) {
1255 free_pcppages_bulk(zone, pcp->count, pcp);
1256 pcp->count = 0;
1257 }
Christoph Lameter3dfa5722008-02-04 22:29:19 -08001258 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001259 }
1260}
Linus Torvalds1da177e2005-04-16 15:20:36 -07001261
Christoph Lameter9f8f2172008-02-04 22:29:11 -08001262/*
1263 * Spill all of this CPU's per-cpu pages back into the buddy allocator.
1264 */
1265void drain_local_pages(void *arg)
1266{
1267 drain_pages(smp_processor_id());
1268}
1269
1270/*
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07001271 * Spill all the per-cpu pages from all CPUs back into the buddy allocator.
1272 *
1273 * Note that this code is protected against sending an IPI to an offline
1274 * CPU but does not guarantee sending an IPI to newly hotplugged CPUs:
1275 * on_each_cpu_mask() blocks hotplug and won't talk to offlined CPUs but
1276 * nothing keeps CPUs from showing up after we populated the cpumask and
1277 * before the call to on_each_cpu_mask().
Christoph Lameter9f8f2172008-02-04 22:29:11 -08001278 */
1279void drain_all_pages(void)
1280{
Gilad Ben-Yossef74046492012-03-28 14:42:45 -07001281 int cpu;
1282 struct per_cpu_pageset *pcp;
1283 struct zone *zone;
1284
1285 /*
1286 * Allocate in the BSS so we wont require allocation in
1287 * direct reclaim path for CONFIG_CPUMASK_OFFSTACK=y
1288 */
1289 static cpumask_t cpus_with_pcps;
1290
1291 /*
1292 * We don't care about racing with CPU hotplug event
1293 * as offline notification will cause the notified
1294 * cpu to drain that CPU pcps and on_each_cpu_mask
1295 * disables preemption as part of its processing
1296 */
1297 for_each_online_cpu(cpu) {
1298 bool has_pcps = false;
1299 for_each_populated_zone(zone) {
1300 pcp = per_cpu_ptr(zone->pageset, cpu);
1301 if (pcp->pcp.count) {
1302 has_pcps = true;
1303 break;
1304 }
1305 }
1306 if (has_pcps)
1307 cpumask_set_cpu(cpu, &cpus_with_pcps);
1308 else
1309 cpumask_clear_cpu(cpu, &cpus_with_pcps);
1310 }
1311 on_each_cpu_mask(&cpus_with_pcps, drain_local_pages, NULL, 1);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08001312}
1313
Rafael J. Wysocki296699d2007-07-29 23:27:18 +02001314#ifdef CONFIG_HIBERNATION
Linus Torvalds1da177e2005-04-16 15:20:36 -07001315
1316void mark_free_pages(struct zone *zone)
1317{
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07001318 unsigned long pfn, max_zone_pfn;
1319 unsigned long flags;
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001320 int order, t;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001321 struct list_head *curr;
1322
1323 if (!zone->spanned_pages)
1324 return;
1325
1326 spin_lock_irqsave(&zone->lock, flags);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07001327
1328 max_zone_pfn = zone->zone_start_pfn + zone->spanned_pages;
1329 for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1330 if (pfn_valid(pfn)) {
1331 struct page *page = pfn_to_page(pfn);
1332
Rafael J. Wysocki7be98232007-05-06 14:50:42 -07001333 if (!swsusp_page_is_forbidden(page))
1334 swsusp_unset_page_free(page);
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07001335 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001336
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001337 for_each_migratetype_order(order, t) {
1338 list_for_each(curr, &zone->free_area[order].free_list[t]) {
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07001339 unsigned long i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001340
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07001341 pfn = page_to_pfn(list_entry(curr, struct page, lru));
1342 for (i = 0; i < (1UL << order); i++)
Rafael J. Wysocki7be98232007-05-06 14:50:42 -07001343 swsusp_set_page_free(pfn_to_page(pfn + i));
Rafael J. Wysockif623f0d2006-09-25 23:32:49 -07001344 }
Mel Gormanb2a0ac82007-10-16 01:25:48 -07001345 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001346 spin_unlock_irqrestore(&zone->lock, flags);
1347}
Mel Gormane2c55dc2007-10-16 01:25:50 -07001348#endif /* CONFIG_PM */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001349
1350/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001351 * Free a 0-order page
Li Hongfc916682010-03-05 13:41:54 -08001352 * cold == 1 ? free a cold page : free a hot page
Linus Torvalds1da177e2005-04-16 15:20:36 -07001353 */
Li Hongfc916682010-03-05 13:41:54 -08001354void free_hot_cold_page(struct page *page, int cold)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001355{
1356 struct zone *zone = page_zone(page);
1357 struct per_cpu_pages *pcp;
1358 unsigned long flags;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001359 int migratetype;
Johannes Weiner451ea252009-09-21 17:01:48 -07001360 int wasMlocked = __TestClearPageMlocked(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001361
KOSAKI Motohiroec95f532010-05-24 14:32:38 -07001362 if (!free_pages_prepare(page, 0))
Hugh Dickins689bceb2005-11-21 21:32:20 -08001363 return;
1364
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001365 migratetype = get_pageblock_migratetype(page);
1366 set_page_private(page, migratetype);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001367 local_irq_save(flags);
Johannes Weinerc2773312009-06-19 19:30:56 +02001368 if (unlikely(wasMlocked))
Mel Gormanda456f12009-06-16 15:32:08 -07001369 free_page_mlock(page);
Christoph Lameterf8891e52006-06-30 01:55:45 -07001370 __count_vm_event(PGFREE);
Mel Gormanda456f12009-06-16 15:32:08 -07001371
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001372 /*
1373 * We only track unmovable, reclaimable and movable on pcp lists.
1374 * Free ISOLATE pages back to the allocator because they are being
1375 * offlined but treat RESERVE as movable pages so we can get those
1376 * areas back if necessary. Otherwise, we may have to free
1377 * excessively into the page allocator
1378 */
1379 if (migratetype >= MIGRATE_PCPTYPES) {
Laura Abbottbe911be2013-02-12 13:30:04 -08001380 if (unlikely(migratetype == MIGRATE_ISOLATE) ||
1381 is_migrate_cma(migratetype)) {
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001382 free_one_page(zone, page, 0, migratetype);
1383 goto out;
1384 }
1385 migratetype = MIGRATE_MOVABLE;
1386 }
1387
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09001388 pcp = &this_cpu_ptr(zone->pageset)->pcp;
Christoph Lameter3dfa5722008-02-04 22:29:19 -08001389 if (cold)
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001390 list_add_tail(&page->lru, &pcp->lists[migratetype]);
Christoph Lameter3dfa5722008-02-04 22:29:19 -08001391 else
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001392 list_add(&page->lru, &pcp->lists[migratetype]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393 pcp->count++;
Nick Piggin48db57f2006-01-08 01:00:42 -08001394 if (pcp->count >= pcp->high) {
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001395 free_pcppages_bulk(zone, pcp->batch, pcp);
Nick Piggin48db57f2006-01-08 01:00:42 -08001396 pcp->count -= pcp->batch;
1397 }
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001398
1399out:
Linus Torvalds1da177e2005-04-16 15:20:36 -07001400 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001401}
1402
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08001403/*
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08001404 * Free a list of 0-order pages
1405 */
1406void free_hot_cold_page_list(struct list_head *list, int cold)
1407{
1408 struct page *page, *next;
1409
1410 list_for_each_entry_safe(page, next, list, lru) {
Konstantin Khlebnikovb413d482012-01-10 15:07:09 -08001411 trace_mm_page_free_batched(page, cold);
Konstantin Khlebnikovcc598502012-01-10 15:07:04 -08001412 free_hot_cold_page(page, cold);
1413 }
1414}
1415
1416/*
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08001417 * split_page takes a non-compound higher-order page, and splits it into
1418 * n (1<<order) sub-pages: page[0..n]
1419 * Each sub-page must be freed individually.
1420 *
1421 * Note: this is probably too low level an operation for use in drivers.
1422 * Please consult with lkml before using this in your driver.
1423 */
1424void split_page(struct page *page, unsigned int order)
1425{
1426 int i;
1427
Nick Piggin725d7042006-09-25 23:30:55 -07001428 VM_BUG_ON(PageCompound(page));
1429 VM_BUG_ON(!page_count(page));
Vegard Nossumb1eeab62008-11-25 16:55:53 +01001430
1431#ifdef CONFIG_KMEMCHECK
1432 /*
1433 * Split shadow pages too, because free(page[0]) would
1434 * otherwise free the whole shadow.
1435 */
1436 if (kmemcheck_page_is_tracked(page))
1437 split_page(virt_to_page(page[0].shadow), order);
1438#endif
1439
Nick Piggin7835e982006-03-22 00:08:40 -08001440 for (i = 1; i < (1 << order); i++)
1441 set_page_refcounted(page + i);
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08001442}
Nick Piggin8dfcc9b2006-03-22 00:08:05 -08001443
Mel Gorman98149402013-01-11 14:32:16 -08001444static int __isolate_free_page(struct page *page, unsigned int order)
Mel Gorman748446b2010-05-24 14:32:27 -07001445{
Mel Gorman748446b2010-05-24 14:32:27 -07001446 unsigned long watermark;
1447 struct zone *zone;
Liam Mark4ca9a212012-11-27 18:49:58 -08001448 int mt;
Mel Gorman748446b2010-05-24 14:32:27 -07001449
1450 BUG_ON(!PageBuddy(page));
1451
1452 zone = page_zone(page);
Liam Mark4ca9a212012-11-27 18:49:58 -08001453 mt = get_pageblock_migratetype(page);
Mel Gorman748446b2010-05-24 14:32:27 -07001454
Marek Szyprowski993e9512012-12-11 16:02:57 -08001455 if (mt != MIGRATE_ISOLATE && !is_migrate_cma(mt)) {
1456 /* Obey watermarks as if the page was being allocated */
1457 watermark = low_wmark_pages(zone) + (1 << order);
1458 if (!zone_watermark_ok(zone, 0, watermark, 0, 0))
1459 return 0;
1460
Mel Gorman98149402013-01-11 14:32:16 -08001461 __mod_zone_freepage_state(zone, -(1UL << order), mt);
Marek Szyprowski993e9512012-12-11 16:02:57 -08001462 }
Mel Gorman748446b2010-05-24 14:32:27 -07001463
1464 /* Remove page from free list */
1465 list_del(&page->lru);
1466 zone->free_area[order].nr_free--;
1467 rmv_page_order(page);
Larry Bassel6d6e2c92012-12-14 14:21:05 -08001468
Mel Gorman98149402013-01-11 14:32:16 -08001469 /* Set the pageblock if the isolated page is at least a pageblock */
Mel Gorman748446b2010-05-24 14:32:27 -07001470 if (order >= pageblock_order - 1) {
1471 struct page *endpage = page + (1 << order) - 1;
Michal Nazarewiczd4158d22011-12-29 13:09:50 +01001472 for (; page < endpage; page += pageblock_nr_pages) {
Liam Mark4ca9a212012-11-27 18:49:58 -08001473 mt = get_pageblock_migratetype(page);
Michal Nazarewiczd4158d22011-12-29 13:09:50 +01001474 if (mt != MIGRATE_ISOLATE && !is_migrate_cma(mt))
1475 set_pageblock_migratetype(page,
1476 MIGRATE_MOVABLE);
1477 }
Mel Gorman748446b2010-05-24 14:32:27 -07001478 }
1479
Mel Gorman0a1104a2012-10-08 16:29:12 -07001480 return 1UL << order;
1481}
1482
1483/*
1484 * Similar to split_page except the page is already free. As this is only
1485 * being used for migration, the migratetype of the block also changes.
1486 * As this is called with interrupts disabled, the caller is responsible
1487 * for calling arch_alloc_page() and kernel_map_page() after interrupts
1488 * are enabled.
1489 *
1490 * Note: this is probably too low level an operation for use in drivers.
1491 * Please consult with lkml before using this in your driver.
1492 */
1493int split_free_page(struct page *page)
1494{
1495 unsigned int order;
1496 int nr_pages;
1497
Mel Gorman0a1104a2012-10-08 16:29:12 -07001498 order = page_order(page);
1499
Mel Gorman98149402013-01-11 14:32:16 -08001500 nr_pages = __isolate_free_page(page, order);
Mel Gorman0a1104a2012-10-08 16:29:12 -07001501 if (!nr_pages)
1502 return 0;
1503
1504 /* Split into individual pages */
1505 set_page_refcounted(page);
1506 split_page(page, order);
1507 return nr_pages;
Mel Gorman748446b2010-05-24 14:32:27 -07001508}
1509
1510/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001511 * Really, prep_compound_page() should be called from __rmqueue_bulk(). But
1512 * we cheat by calling it from here, in the order > 0 path. Saves a branch
1513 * or two.
1514 */
Mel Gorman0a15c3e2009-06-16 15:32:05 -07001515static inline
1516struct page *buffered_rmqueue(struct zone *preferred_zone,
Mel Gorman3dd28262009-06-16 15:32:00 -07001517 struct zone *zone, int order, gfp_t gfp_flags,
1518 int migratetype)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001519{
1520 unsigned long flags;
Hugh Dickins689bceb2005-11-21 21:32:20 -08001521 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001522 int cold = !!(gfp_flags & __GFP_COLD);
1523
Hugh Dickins689bceb2005-11-21 21:32:20 -08001524again:
Nick Piggin48db57f2006-01-08 01:00:42 -08001525 if (likely(order == 0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001526 struct per_cpu_pages *pcp;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001527 struct list_head *list;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001528
Linus Torvalds1da177e2005-04-16 15:20:36 -07001529 local_irq_save(flags);
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09001530 pcp = &this_cpu_ptr(zone->pageset)->pcp;
1531 list = &pcp->lists[migratetype];
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001532 if (list_empty(list)) {
Mel Gorman535131e2007-10-16 01:25:49 -07001533 pcp->count += rmqueue_bulk(zone, 0,
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001534 pcp->batch, list,
Heesub Shin771aaa62013-01-07 11:10:13 +09001535 migratetype, cold,
1536 gfp_flags & __GFP_CMA);
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001537 if (unlikely(list_empty(list)))
Shaohua Li6fb332f2009-09-21 17:01:17 -07001538 goto failed;
Mel Gorman535131e2007-10-16 01:25:49 -07001539 }
Mel Gormanb92a6ed2007-10-16 01:25:50 -07001540
Mel Gorman5f8dcc22009-09-21 17:03:19 -07001541 if (cold)
1542 page = list_entry(list->prev, struct page, lru);
1543 else
1544 page = list_entry(list->next, struct page, lru);
1545
Mel Gormanb92a6ed2007-10-16 01:25:50 -07001546 list_del(&page->lru);
1547 pcp->count--;
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001548 } else {
Andrew Mortondab48da2009-06-16 15:32:37 -07001549 if (unlikely(gfp_flags & __GFP_NOFAIL)) {
1550 /*
1551 * __GFP_NOFAIL is not to be used in new code.
1552 *
1553 * All __GFP_NOFAIL callers should be fixed so that they
1554 * properly detect and handle allocation failures.
1555 *
1556 * We most definitely don't want callers attempting to
Linus Torvalds4923abf2009-06-24 12:16:49 -07001557 * allocate greater than order-1 page units with
Andrew Mortondab48da2009-06-16 15:32:37 -07001558 * __GFP_NOFAIL.
1559 */
Linus Torvalds4923abf2009-06-24 12:16:49 -07001560 WARN_ON_ONCE(order > 1);
Andrew Mortondab48da2009-06-16 15:32:37 -07001561 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001562 spin_lock_irqsave(&zone->lock, flags);
Heesub Shin771aaa62013-01-07 11:10:13 +09001563 if (gfp_flags & __GFP_CMA)
1564 page = __rmqueue_cma(zone, order, migratetype);
1565 else
1566 page = __rmqueue(zone, order, migratetype);
Nick Piggina74609f2006-01-06 00:11:20 -08001567 spin_unlock(&zone->lock);
1568 if (!page)
1569 goto failed;
Larry Bassel6d6e2c92012-12-14 14:21:05 -08001570 __mod_zone_freepage_state(zone, -(1 << order),
1571 get_pageblock_migratetype(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -07001572 }
1573
Christoph Lameterf8891e52006-06-30 01:55:45 -07001574 __count_zone_vm_events(PGALLOC, zone, 1 << order);
Andi Kleen78afd562011-03-22 16:33:12 -07001575 zone_statistics(preferred_zone, zone, gfp_flags);
Nick Piggina74609f2006-01-06 00:11:20 -08001576 local_irq_restore(flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001577
Nick Piggin725d7042006-09-25 23:30:55 -07001578 VM_BUG_ON(bad_range(zone, page));
Nick Piggin17cf4402006-03-22 00:08:41 -08001579 if (prep_new_page(page, order, gfp_flags))
Nick Piggina74609f2006-01-06 00:11:20 -08001580 goto again;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001581 return page;
Nick Piggina74609f2006-01-06 00:11:20 -08001582
1583failed:
1584 local_irq_restore(flags);
Nick Piggina74609f2006-01-06 00:11:20 -08001585 return NULL;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001586}
1587
Akinobu Mita933e3122006-12-08 02:39:45 -08001588#ifdef CONFIG_FAIL_PAGE_ALLOC
1589
Akinobu Mitab2588c42011-07-26 16:09:03 -07001590static struct {
Akinobu Mita933e3122006-12-08 02:39:45 -08001591 struct fault_attr attr;
1592
1593 u32 ignore_gfp_highmem;
1594 u32 ignore_gfp_wait;
Akinobu Mita54114992007-07-15 23:40:23 -07001595 u32 min_order;
Akinobu Mita933e3122006-12-08 02:39:45 -08001596} fail_page_alloc = {
1597 .attr = FAULT_ATTR_INITIALIZER,
Don Mullis6b1b60f2006-12-08 02:39:53 -08001598 .ignore_gfp_wait = 1,
1599 .ignore_gfp_highmem = 1,
Akinobu Mita54114992007-07-15 23:40:23 -07001600 .min_order = 1,
Akinobu Mita933e3122006-12-08 02:39:45 -08001601};
1602
1603static int __init setup_fail_page_alloc(char *str)
1604{
1605 return setup_fault_attr(&fail_page_alloc.attr, str);
1606}
1607__setup("fail_page_alloc=", setup_fail_page_alloc);
1608
1609static int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
1610{
Akinobu Mita54114992007-07-15 23:40:23 -07001611 if (order < fail_page_alloc.min_order)
1612 return 0;
Akinobu Mita933e3122006-12-08 02:39:45 -08001613 if (gfp_mask & __GFP_NOFAIL)
1614 return 0;
1615 if (fail_page_alloc.ignore_gfp_highmem && (gfp_mask & __GFP_HIGHMEM))
1616 return 0;
1617 if (fail_page_alloc.ignore_gfp_wait && (gfp_mask & __GFP_WAIT))
1618 return 0;
1619
1620 return should_fail(&fail_page_alloc.attr, 1 << order);
1621}
1622
1623#ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
1624
1625static int __init fail_page_alloc_debugfs(void)
1626{
Al Virof4ae40a2011-07-24 04:33:43 -04001627 umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
Akinobu Mita933e3122006-12-08 02:39:45 -08001628 struct dentry *dir;
Akinobu Mita933e3122006-12-08 02:39:45 -08001629
Akinobu Mitadd48c082011-08-03 16:21:01 -07001630 dir = fault_create_debugfs_attr("fail_page_alloc", NULL,
1631 &fail_page_alloc.attr);
1632 if (IS_ERR(dir))
1633 return PTR_ERR(dir);
Akinobu Mita933e3122006-12-08 02:39:45 -08001634
Akinobu Mitab2588c42011-07-26 16:09:03 -07001635 if (!debugfs_create_bool("ignore-gfp-wait", mode, dir,
1636 &fail_page_alloc.ignore_gfp_wait))
1637 goto fail;
1638 if (!debugfs_create_bool("ignore-gfp-highmem", mode, dir,
1639 &fail_page_alloc.ignore_gfp_highmem))
1640 goto fail;
1641 if (!debugfs_create_u32("min-order", mode, dir,
1642 &fail_page_alloc.min_order))
1643 goto fail;
Akinobu Mita933e3122006-12-08 02:39:45 -08001644
Akinobu Mitab2588c42011-07-26 16:09:03 -07001645 return 0;
1646fail:
Akinobu Mitadd48c082011-08-03 16:21:01 -07001647 debugfs_remove_recursive(dir);
Akinobu Mita933e3122006-12-08 02:39:45 -08001648
Akinobu Mitab2588c42011-07-26 16:09:03 -07001649 return -ENOMEM;
Akinobu Mita933e3122006-12-08 02:39:45 -08001650}
1651
1652late_initcall(fail_page_alloc_debugfs);
1653
1654#endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
1655
1656#else /* CONFIG_FAIL_PAGE_ALLOC */
1657
1658static inline int should_fail_alloc_page(gfp_t gfp_mask, unsigned int order)
1659{
1660 return 0;
1661}
1662
1663#endif /* CONFIG_FAIL_PAGE_ALLOC */
1664
Linus Torvalds1da177e2005-04-16 15:20:36 -07001665/*
Mel Gorman88f5acf2011-01-13 15:45:41 -08001666 * Return true if free pages are above 'mark'. This takes into account the order
Linus Torvalds1da177e2005-04-16 15:20:36 -07001667 * of the allocation.
1668 */
Mel Gorman88f5acf2011-01-13 15:45:41 -08001669static bool __zone_watermark_ok(struct zone *z, int order, unsigned long mark,
1670 int classzone_idx, int alloc_flags, long free_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001671{
Jack Cheung9f41da82011-11-28 16:41:28 -08001672 /* free_pages may go negative - that's OK */
Christoph Lameterd23ad422007-02-10 01:43:02 -08001673 long min = mark;
Jack Cheung9f41da82011-11-28 16:41:28 -08001674 long lowmem_reserve = z->lowmem_reserve[classzone_idx];
Linus Torvalds1da177e2005-04-16 15:20:36 -07001675 int o;
Tomasz Stanislawski18d4e2d2013-05-09 09:50:46 +02001676 long free_cma = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001677
Michal Hockodf0a6da2012-01-10 15:08:02 -08001678 free_pages -= (1 << order) - 1;
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001679 if (alloc_flags & ALLOC_HIGH)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001680 min -= min / 2;
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001681 if (alloc_flags & ALLOC_HARDER)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001682 min -= min / 4;
Bartlomiej Zolnierkiewicz19644312012-10-08 16:32:05 -07001683#ifdef CONFIG_CMA
1684 /* If allocation can't use CMA areas don't use free CMA pages */
1685 if (!(alloc_flags & ALLOC_CMA))
Tomasz Stanislawski18d4e2d2013-05-09 09:50:46 +02001686 free_cma = zone_page_state(z, NR_FREE_CMA_PAGES);
Bartlomiej Zolnierkiewicz19644312012-10-08 16:32:05 -07001687#endif
Tomasz Stanislawski18d4e2d2013-05-09 09:50:46 +02001688
1689 if (free_pages - free_cma <= min + lowmem_reserve)
Mel Gorman88f5acf2011-01-13 15:45:41 -08001690 return false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001691 for (o = 0; o < order; o++) {
1692 /* At the next order, this order's pages become unavailable */
1693 free_pages -= z->free_area[o].nr_free << o;
1694
1695 /* Require fewer higher order pages to be free */
Arve Hjønnevågcde89492009-02-17 14:51:02 -08001696 min >>= min_free_order_shift;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001697
1698 if (free_pages <= min)
Mel Gorman88f5acf2011-01-13 15:45:41 -08001699 return false;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001700 }
Mel Gorman88f5acf2011-01-13 15:45:41 -08001701 return true;
1702}
1703
1704bool zone_watermark_ok(struct zone *z, int order, unsigned long mark,
1705 int classzone_idx, int alloc_flags)
1706{
1707 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
1708 zone_page_state(z, NR_FREE_PAGES));
1709}
1710
1711bool zone_watermark_ok_safe(struct zone *z, int order, unsigned long mark,
1712 int classzone_idx, int alloc_flags)
1713{
1714 long free_pages = zone_page_state(z, NR_FREE_PAGES);
1715
1716 if (z->percpu_drift_mark && free_pages < z->percpu_drift_mark)
1717 free_pages = zone_page_state_snapshot(z, NR_FREE_PAGES);
1718
1719 return __zone_watermark_ok(z, order, mark, classzone_idx, alloc_flags,
1720 free_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001721}
1722
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001723#ifdef CONFIG_NUMA
1724/*
1725 * zlc_setup - Setup for "zonelist cache". Uses cached zone data to
1726 * skip over zones that are not allowed by the cpuset, or that have
1727 * been recently (in last second) found to be nearly full. See further
1728 * comments in mmzone.h. Reduces cache footprint of zonelist scans
Simon Arlott183ff222007-10-20 01:27:18 +02001729 * that have to skip over a lot of full or unallowed zones.
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001730 *
1731 * If the zonelist cache is present in the passed in zonelist, then
1732 * returns a pointer to the allowed node mask (either the current
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07001733 * tasks mems_allowed, or node_states[N_HIGH_MEMORY].)
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001734 *
1735 * If the zonelist cache is not available for this zonelist, does
1736 * nothing and returns NULL.
1737 *
1738 * If the fullzones BITMAP in the zonelist cache is stale (more than
1739 * a second since last zap'd) then we zap it out (clear its bits.)
1740 *
1741 * We hold off even calling zlc_setup, until after we've checked the
1742 * first zone in the zonelist, on the theory that most allocations will
1743 * be satisfied from that first zone, so best to examine that zone as
1744 * quickly as we can.
1745 */
1746static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
1747{
1748 struct zonelist_cache *zlc; /* cached zonelist speedup info */
1749 nodemask_t *allowednodes; /* zonelist_cache approximation */
1750
1751 zlc = zonelist->zlcache_ptr;
1752 if (!zlc)
1753 return NULL;
1754
S.Caglar Onurf05111f2008-04-28 02:12:38 -07001755 if (time_after(jiffies, zlc->last_full_zap + HZ)) {
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001756 bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
1757 zlc->last_full_zap = jiffies;
1758 }
1759
1760 allowednodes = !in_interrupt() && (alloc_flags & ALLOC_CPUSET) ?
1761 &cpuset_current_mems_allowed :
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07001762 &node_states[N_HIGH_MEMORY];
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001763 return allowednodes;
1764}
1765
1766/*
1767 * Given 'z' scanning a zonelist, run a couple of quick checks to see
1768 * if it is worth looking at further for free memory:
1769 * 1) Check that the zone isn't thought to be full (doesn't have its
1770 * bit set in the zonelist_cache fullzones BITMAP).
1771 * 2) Check that the zones node (obtained from the zonelist_cache
1772 * z_to_n[] mapping) is allowed in the passed in allowednodes mask.
1773 * Return true (non-zero) if zone is worth looking at further, or
1774 * else return false (zero) if it is not.
1775 *
1776 * This check -ignores- the distinction between various watermarks,
1777 * such as GFP_HIGH, GFP_ATOMIC, PF_MEMALLOC, ... If a zone is
1778 * found to be full for any variation of these watermarks, it will
1779 * be considered full for up to one second by all requests, unless
1780 * we are so low on memory on all allowed nodes that we are forced
1781 * into the second scan of the zonelist.
1782 *
1783 * In the second scan we ignore this zonelist cache and exactly
1784 * apply the watermarks to all zones, even it is slower to do so.
1785 * We are low on memory in the second scan, and should leave no stone
1786 * unturned looking for a free page.
1787 */
Mel Gormandd1a2392008-04-28 02:12:17 -07001788static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001789 nodemask_t *allowednodes)
1790{
1791 struct zonelist_cache *zlc; /* cached zonelist speedup info */
1792 int i; /* index of *z in zonelist zones */
1793 int n; /* node that zone *z is on */
1794
1795 zlc = zonelist->zlcache_ptr;
1796 if (!zlc)
1797 return 1;
1798
Mel Gormandd1a2392008-04-28 02:12:17 -07001799 i = z - zonelist->_zonerefs;
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001800 n = zlc->z_to_n[i];
1801
1802 /* This zone is worth trying if it is allowed but not full */
1803 return node_isset(n, *allowednodes) && !test_bit(i, zlc->fullzones);
1804}
1805
1806/*
1807 * Given 'z' scanning a zonelist, set the corresponding bit in
1808 * zlc->fullzones, so that subsequent attempts to allocate a page
1809 * from that zone don't waste time re-examining it.
1810 */
Mel Gormandd1a2392008-04-28 02:12:17 -07001811static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001812{
1813 struct zonelist_cache *zlc; /* cached zonelist speedup info */
1814 int i; /* index of *z in zonelist zones */
1815
1816 zlc = zonelist->zlcache_ptr;
1817 if (!zlc)
1818 return;
1819
Mel Gormandd1a2392008-04-28 02:12:17 -07001820 i = z - zonelist->_zonerefs;
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001821
1822 set_bit(i, zlc->fullzones);
1823}
1824
Mel Gorman76d3fbf2011-07-25 17:12:30 -07001825/*
1826 * clear all zones full, called after direct reclaim makes progress so that
1827 * a zone that was recently full is not skipped over for up to a second
1828 */
1829static void zlc_clear_zones_full(struct zonelist *zonelist)
1830{
1831 struct zonelist_cache *zlc; /* cached zonelist speedup info */
1832
1833 zlc = zonelist->zlcache_ptr;
1834 if (!zlc)
1835 return;
1836
1837 bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
1838}
1839
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001840#else /* CONFIG_NUMA */
1841
1842static nodemask_t *zlc_setup(struct zonelist *zonelist, int alloc_flags)
1843{
1844 return NULL;
1845}
1846
Mel Gormandd1a2392008-04-28 02:12:17 -07001847static int zlc_zone_worth_trying(struct zonelist *zonelist, struct zoneref *z,
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001848 nodemask_t *allowednodes)
1849{
1850 return 1;
1851}
1852
Mel Gormandd1a2392008-04-28 02:12:17 -07001853static void zlc_mark_zone_full(struct zonelist *zonelist, struct zoneref *z)
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001854{
1855}
Mel Gorman76d3fbf2011-07-25 17:12:30 -07001856
1857static void zlc_clear_zones_full(struct zonelist *zonelist)
1858{
1859}
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001860#endif /* CONFIG_NUMA */
1861
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001862/*
Paul Jackson0798e512006-12-06 20:31:38 -08001863 * get_page_from_freelist goes through the zonelist trying to allocate
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001864 * a page.
1865 */
1866static struct page *
Mel Gorman19770b32008-04-28 02:12:18 -07001867get_page_from_freelist(gfp_t gfp_mask, nodemask_t *nodemask, unsigned int order,
Mel Gorman5117f452009-06-16 15:31:59 -07001868 struct zonelist *zonelist, int high_zoneidx, int alloc_flags,
Mel Gorman3dd28262009-06-16 15:32:00 -07001869 struct zone *preferred_zone, int migratetype)
Martin Hicks753ee722005-06-21 17:14:41 -07001870{
Mel Gormandd1a2392008-04-28 02:12:17 -07001871 struct zoneref *z;
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001872 struct page *page = NULL;
Mel Gorman54a6eb52008-04-28 02:12:16 -07001873 int classzone_idx;
Mel Gorman5117f452009-06-16 15:31:59 -07001874 struct zone *zone;
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001875 nodemask_t *allowednodes = NULL;/* zonelist_cache approximation */
1876 int zlc_active = 0; /* set if using zonelist_cache */
1877 int did_zlc_setup = 0; /* just call zlc_setup() one time */
Mel Gorman54a6eb52008-04-28 02:12:16 -07001878
Mel Gorman19770b32008-04-28 02:12:18 -07001879 classzone_idx = zone_idx(preferred_zone);
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001880zonelist_scan:
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001881 /*
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001882 * Scan zonelist, looking for a zone with enough free.
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001883 * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
1884 */
Mel Gorman19770b32008-04-28 02:12:18 -07001885 for_each_zone_zonelist_nodemask(zone, z, zonelist,
1886 high_zoneidx, nodemask) {
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001887 if (NUMA_BUILD && zlc_active &&
1888 !zlc_zone_worth_trying(zonelist, z, allowednodes))
1889 continue;
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001890 if ((alloc_flags & ALLOC_CPUSET) &&
Paul Jackson02a0e532006-12-13 00:34:25 -08001891 !cpuset_zone_allowed_softwall(zone, gfp_mask))
Mel Gormancd38b112011-07-25 17:12:29 -07001892 continue;
Johannes Weinera756cf52012-01-10 15:07:49 -08001893 /*
1894 * When allocating a page cache page for writing, we
1895 * want to get it from a zone that is within its dirty
1896 * limit, such that no single zone holds more than its
1897 * proportional share of globally allowed dirty pages.
1898 * The dirty limits take into account the zone's
1899 * lowmem reserves and high watermark so that kswapd
1900 * should be able to balance it without having to
1901 * write pages from its LRU list.
1902 *
1903 * This may look like it could increase pressure on
1904 * lower zones by failing allocations in higher zones
1905 * before they are full. But the pages that do spill
1906 * over are limited as the lower zones are protected
1907 * by this very same mechanism. It should not become
1908 * a practical burden to them.
1909 *
1910 * XXX: For now, allow allocations to potentially
1911 * exceed the per-zone dirty limit in the slowpath
1912 * (ALLOC_WMARK_LOW unset) before going into reclaim,
1913 * which is important when on a NUMA setup the allowed
1914 * zones are together not big enough to reach the
1915 * global limit. The proper fix for these situations
1916 * will require awareness of zones in the
1917 * dirty-throttling and the flusher threads.
1918 */
1919 if ((alloc_flags & ALLOC_WMARK_LOW) &&
1920 (gfp_mask & __GFP_WRITE) && !zone_dirty_ok(zone))
1921 goto this_zone_full;
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001922
Mel Gorman41858962009-06-16 15:32:12 -07001923 BUILD_BUG_ON(ALLOC_NO_WATERMARKS < NR_WMARK);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001924 if (!(alloc_flags & ALLOC_NO_WATERMARKS)) {
Nick Piggin31488902005-11-28 13:44:03 -08001925 unsigned long mark;
Mel Gormanfa5e0842009-06-16 15:33:22 -07001926 int ret;
1927
Mel Gorman41858962009-06-16 15:32:12 -07001928 mark = zone->watermark[alloc_flags & ALLOC_WMARK_MASK];
Mel Gormanfa5e0842009-06-16 15:33:22 -07001929 if (zone_watermark_ok(zone, order, mark,
1930 classzone_idx, alloc_flags))
1931 goto try_this_zone;
1932
Mel Gormancd38b112011-07-25 17:12:29 -07001933 if (NUMA_BUILD && !did_zlc_setup && nr_online_nodes > 1) {
1934 /*
1935 * we do zlc_setup if there are multiple nodes
1936 * and before considering the first zone allowed
1937 * by the cpuset.
1938 */
1939 allowednodes = zlc_setup(zonelist, alloc_flags);
1940 zlc_active = 1;
1941 did_zlc_setup = 1;
1942 }
1943
Mel Gormanfa5e0842009-06-16 15:33:22 -07001944 if (zone_reclaim_mode == 0)
1945 goto this_zone_full;
1946
Mel Gormancd38b112011-07-25 17:12:29 -07001947 /*
1948 * As we may have just activated ZLC, check if the first
1949 * eligible zone has failed zone_reclaim recently.
1950 */
1951 if (NUMA_BUILD && zlc_active &&
1952 !zlc_zone_worth_trying(zonelist, z, allowednodes))
1953 continue;
1954
Mel Gormanfa5e0842009-06-16 15:33:22 -07001955 ret = zone_reclaim(zone, gfp_mask, order);
1956 switch (ret) {
1957 case ZONE_RECLAIM_NOSCAN:
1958 /* did not scan */
Mel Gormancd38b112011-07-25 17:12:29 -07001959 continue;
Mel Gormanfa5e0842009-06-16 15:33:22 -07001960 case ZONE_RECLAIM_FULL:
1961 /* scanned but unreclaimable */
Mel Gormancd38b112011-07-25 17:12:29 -07001962 continue;
Mel Gormanfa5e0842009-06-16 15:33:22 -07001963 default:
1964 /* did we reclaim enough */
1965 if (!zone_watermark_ok(zone, order, mark,
1966 classzone_idx, alloc_flags))
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001967 goto this_zone_full;
Paul Jackson0798e512006-12-06 20:31:38 -08001968 }
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001969 }
1970
Mel Gormanfa5e0842009-06-16 15:33:22 -07001971try_this_zone:
Mel Gorman3dd28262009-06-16 15:32:00 -07001972 page = buffered_rmqueue(preferred_zone, zone, order,
1973 gfp_mask, migratetype);
Paul Jackson0798e512006-12-06 20:31:38 -08001974 if (page)
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001975 break;
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001976this_zone_full:
1977 if (NUMA_BUILD)
1978 zlc_mark_zone_full(zonelist, z);
Mel Gorman54a6eb52008-04-28 02:12:16 -07001979 }
Paul Jackson9276b1bc2006-12-06 20:31:48 -08001980
1981 if (unlikely(NUMA_BUILD && page == NULL && zlc_active)) {
1982 /* Disable zlc cache for second zonelist scan */
1983 zlc_active = 0;
1984 goto zonelist_scan;
1985 }
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08001986 return page;
Martin Hicks753ee722005-06-21 17:14:41 -07001987}
1988
David Rientjes29423e72011-03-22 16:30:47 -07001989/*
1990 * Large machines with many possible nodes should not always dump per-node
1991 * meminfo in irq context.
1992 */
1993static inline bool should_suppress_show_mem(void)
1994{
1995 bool ret = false;
1996
1997#if NODES_SHIFT > 8
1998 ret = in_interrupt();
1999#endif
2000 return ret;
2001}
2002
Dave Hansena238ab52011-05-24 17:12:16 -07002003static DEFINE_RATELIMIT_STATE(nopage_rs,
2004 DEFAULT_RATELIMIT_INTERVAL,
2005 DEFAULT_RATELIMIT_BURST);
2006
2007void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...)
2008{
Dave Hansena238ab52011-05-24 17:12:16 -07002009 unsigned int filter = SHOW_MEM_FILTER_NODES;
2010
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -08002011 if ((gfp_mask & __GFP_NOWARN) || !__ratelimit(&nopage_rs) ||
2012 debug_guardpage_minorder() > 0)
Dave Hansena238ab52011-05-24 17:12:16 -07002013 return;
2014
2015 /*
2016 * This documents exceptions given to allocations in certain
2017 * contexts that are allowed to allocate outside current's set
2018 * of allowed nodes.
2019 */
2020 if (!(gfp_mask & __GFP_NOMEMALLOC))
2021 if (test_thread_flag(TIF_MEMDIE) ||
2022 (current->flags & (PF_MEMALLOC | PF_EXITING)))
2023 filter &= ~SHOW_MEM_FILTER_NODES;
2024 if (in_interrupt() || !(gfp_mask & __GFP_WAIT))
2025 filter &= ~SHOW_MEM_FILTER_NODES;
2026
2027 if (fmt) {
Joe Perches3ee9a4f2011-10-31 17:08:35 -07002028 struct va_format vaf;
2029 va_list args;
2030
Dave Hansena238ab52011-05-24 17:12:16 -07002031 va_start(args, fmt);
Joe Perches3ee9a4f2011-10-31 17:08:35 -07002032
2033 vaf.fmt = fmt;
2034 vaf.va = &args;
2035
2036 pr_warn("%pV", &vaf);
2037
Dave Hansena238ab52011-05-24 17:12:16 -07002038 va_end(args);
2039 }
2040
Joe Perches3ee9a4f2011-10-31 17:08:35 -07002041 pr_warn("%s: page allocation failure: order:%d, mode:0x%x\n",
2042 current->comm, order, gfp_mask);
Dave Hansena238ab52011-05-24 17:12:16 -07002043
2044 dump_stack();
2045 if (!should_suppress_show_mem())
2046 show_mem(filter);
2047}
2048
Mel Gorman11e33f62009-06-16 15:31:57 -07002049static inline int
2050should_alloc_retry(gfp_t gfp_mask, unsigned int order,
Mel Gormanf90ac392012-01-10 15:07:15 -08002051 unsigned long did_some_progress,
Mel Gorman11e33f62009-06-16 15:31:57 -07002052 unsigned long pages_reclaimed)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002053{
Mel Gorman11e33f62009-06-16 15:31:57 -07002054 /* Do not loop if specifically requested */
2055 if (gfp_mask & __GFP_NORETRY)
2056 return 0;
2057
Mel Gormanf90ac392012-01-10 15:07:15 -08002058 /* Always retry if specifically requested */
2059 if (gfp_mask & __GFP_NOFAIL)
2060 return 1;
2061
2062 /*
2063 * Suspend converts GFP_KERNEL to __GFP_WAIT which can prevent reclaim
2064 * making forward progress without invoking OOM. Suspend also disables
2065 * storage devices so kswapd will not help. Bail if we are suspending.
2066 */
2067 if (!did_some_progress && pm_suspended_storage())
2068 return 0;
2069
Mel Gorman11e33f62009-06-16 15:31:57 -07002070 /*
2071 * In this implementation, order <= PAGE_ALLOC_COSTLY_ORDER
2072 * means __GFP_NOFAIL, but that may not be true in other
2073 * implementations.
2074 */
2075 if (order <= PAGE_ALLOC_COSTLY_ORDER)
2076 return 1;
2077
2078 /*
2079 * For order > PAGE_ALLOC_COSTLY_ORDER, if __GFP_REPEAT is
2080 * specified, then we retry until we no longer reclaim any pages
2081 * (above), or we've reclaimed an order of pages at least as
2082 * large as the allocation's order. In both cases, if the
2083 * allocation still fails, we stop retrying.
2084 */
2085 if (gfp_mask & __GFP_REPEAT && pages_reclaimed < (1 << order))
2086 return 1;
2087
Mel Gorman11e33f62009-06-16 15:31:57 -07002088 return 0;
2089}
2090
2091static inline struct page *
2092__alloc_pages_may_oom(gfp_t gfp_mask, unsigned int order,
2093 struct zonelist *zonelist, enum zone_type high_zoneidx,
Mel Gorman3dd28262009-06-16 15:32:00 -07002094 nodemask_t *nodemask, struct zone *preferred_zone,
2095 int migratetype)
Mel Gorman11e33f62009-06-16 15:31:57 -07002096{
Linus Torvalds1da177e2005-04-16 15:20:36 -07002097 struct page *page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002098
Mel Gorman11e33f62009-06-16 15:31:57 -07002099 /* Acquire the OOM killer lock for the zones in zonelist */
Minchan Kimff321fe2010-08-09 17:18:57 -07002100 if (!try_set_zonelist_oom(zonelist, gfp_mask)) {
Mel Gorman11e33f62009-06-16 15:31:57 -07002101 schedule_timeout_uninterruptible(1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002102 return NULL;
2103 }
Jens Axboe6b1de912005-11-17 21:35:02 +01002104
Mel Gorman11e33f62009-06-16 15:31:57 -07002105 /*
2106 * Go through the zonelist yet one more time, keep very high watermark
2107 * here, this is only to catch a parallel oom killing, we must fail if
2108 * we're still under heavy pressure.
2109 */
2110 page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask,
2111 order, zonelist, high_zoneidx,
Mel Gorman5117f452009-06-16 15:31:59 -07002112 ALLOC_WMARK_HIGH|ALLOC_CPUSET,
Mel Gorman3dd28262009-06-16 15:32:00 -07002113 preferred_zone, migratetype);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08002114 if (page)
Mel Gorman11e33f62009-06-16 15:31:57 -07002115 goto out;
2116
KAMEZAWA Hiroyuki4365a562009-12-15 16:45:33 -08002117 if (!(gfp_mask & __GFP_NOFAIL)) {
2118 /* The OOM killer will not help higher order allocs */
2119 if (order > PAGE_ALLOC_COSTLY_ORDER)
2120 goto out;
David Rientjes03668b32010-08-09 17:18:54 -07002121 /* The OOM killer does not needlessly kill tasks for lowmem */
2122 if (high_zoneidx < ZONE_NORMAL)
2123 goto out;
KAMEZAWA Hiroyuki4365a562009-12-15 16:45:33 -08002124 /*
2125 * GFP_THISNODE contains __GFP_NORETRY and we never hit this.
2126 * Sanity check for bare calls of __GFP_THISNODE, not real OOM.
2127 * The caller should handle page allocation failure by itself if
2128 * it specifies __GFP_THISNODE.
2129 * Note: Hugepage uses it but will hit PAGE_ALLOC_COSTLY_ORDER.
2130 */
2131 if (gfp_mask & __GFP_THISNODE)
2132 goto out;
2133 }
Mel Gorman11e33f62009-06-16 15:31:57 -07002134 /* Exhausted what can be done so it's blamo time */
David Rientjes08ab9b12012-03-21 16:34:04 -07002135 out_of_memory(zonelist, gfp_mask, order, nodemask, false);
Mel Gorman11e33f62009-06-16 15:31:57 -07002136
2137out:
2138 clear_zonelist_oom(zonelist, gfp_mask);
2139 return page;
2140}
2141
Mel Gorman56de7262010-05-24 14:32:30 -07002142#ifdef CONFIG_COMPACTION
2143/* Try memory compaction for high-order allocations before reclaim */
2144static struct page *
2145__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
2146 struct zonelist *zonelist, enum zone_type high_zoneidx,
2147 nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
Mel Gorman66199712012-01-12 17:19:41 -08002148 int migratetype, bool sync_migration,
Mel Gorman27d1af82012-08-21 16:16:17 -07002149 bool *contended_compaction, bool *deferred_compaction,
Mel Gorman66199712012-01-12 17:19:41 -08002150 unsigned long *did_some_progress)
Mel Gorman56de7262010-05-24 14:32:30 -07002151{
Mel Gorman66199712012-01-12 17:19:41 -08002152 if (!order)
Mel Gorman56de7262010-05-24 14:32:30 -07002153 return NULL;
2154
Rik van Rielaff62242012-03-21 16:33:52 -07002155 if (compaction_deferred(preferred_zone, order)) {
Mel Gorman66199712012-01-12 17:19:41 -08002156 *deferred_compaction = true;
2157 return NULL;
2158 }
2159
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08002160 current->flags |= PF_MEMALLOC;
Mel Gorman56de7262010-05-24 14:32:30 -07002161 *did_some_progress = try_to_compact_pages(zonelist, order, gfp_mask,
Mel Gorman27d1af82012-08-21 16:16:17 -07002162 nodemask, sync_migration,
Mel Gorman98149402013-01-11 14:32:16 -08002163 contended_compaction);
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08002164 current->flags &= ~PF_MEMALLOC;
Mel Gorman56de7262010-05-24 14:32:30 -07002165
Mel Gorman0a1104a2012-10-08 16:29:12 -07002166 if (*did_some_progress != COMPACT_SKIPPED) {
Mel Gorman98149402013-01-11 14:32:16 -08002167 struct page *page;
2168
Mel Gorman56de7262010-05-24 14:32:30 -07002169 /* Page migration frees to the PCP lists but we want merging */
2170 drain_pages(get_cpu());
2171 put_cpu();
2172
2173 page = get_page_from_freelist(gfp_mask, nodemask,
2174 order, zonelist, high_zoneidx,
2175 alloc_flags, preferred_zone,
2176 migratetype);
2177 if (page) {
Mel Gorman9317a2b2012-10-08 16:32:47 -07002178 preferred_zone->compact_blockskip_flush = false;
Mel Gorman4f92e252010-05-24 14:32:32 -07002179 preferred_zone->compact_considered = 0;
2180 preferred_zone->compact_defer_shift = 0;
Rik van Rielaff62242012-03-21 16:33:52 -07002181 if (order >= preferred_zone->compact_order_failed)
2182 preferred_zone->compact_order_failed = order + 1;
Mel Gorman56de7262010-05-24 14:32:30 -07002183 count_vm_event(COMPACTSUCCESS);
2184 return page;
2185 }
2186
2187 /*
2188 * It's bad if compaction run occurs and fails.
2189 * The most likely reason is that pages exist,
2190 * but not enough to satisfy watermarks.
2191 */
2192 count_vm_event(COMPACTFAIL);
Mel Gorman66199712012-01-12 17:19:41 -08002193
2194 /*
2195 * As async compaction considers a subset of pageblocks, only
2196 * defer if the failure was a sync compaction failure.
2197 */
2198 if (sync_migration)
Rik van Rielaff62242012-03-21 16:33:52 -07002199 defer_compaction(preferred_zone, order);
Mel Gorman56de7262010-05-24 14:32:30 -07002200
2201 cond_resched();
2202 }
2203
2204 return NULL;
2205}
2206#else
2207static inline struct page *
2208__alloc_pages_direct_compact(gfp_t gfp_mask, unsigned int order,
2209 struct zonelist *zonelist, enum zone_type high_zoneidx,
2210 nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
Mel Gorman66199712012-01-12 17:19:41 -08002211 int migratetype, bool sync_migration,
Mel Gorman27d1af82012-08-21 16:16:17 -07002212 bool *contended_compaction, bool *deferred_compaction,
Mel Gorman66199712012-01-12 17:19:41 -08002213 unsigned long *did_some_progress)
Mel Gorman56de7262010-05-24 14:32:30 -07002214{
2215 return NULL;
2216}
2217#endif /* CONFIG_COMPACTION */
2218
Marek Szyprowski1ed36512012-01-25 12:09:52 +01002219/* Perform direct synchronous page reclaim */
2220static int
2221__perform_reclaim(gfp_t gfp_mask, unsigned int order, struct zonelist *zonelist,
2222 nodemask_t *nodemask)
Mel Gorman11e33f62009-06-16 15:31:57 -07002223{
Mel Gorman11e33f62009-06-16 15:31:57 -07002224 struct reclaim_state reclaim_state;
Marek Szyprowski1ed36512012-01-25 12:09:52 +01002225 int progress;
Mel Gorman11e33f62009-06-16 15:31:57 -07002226
2227 cond_resched();
2228
2229 /* We now go into synchronous reclaim */
2230 cpuset_memory_pressure_bump();
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08002231 current->flags |= PF_MEMALLOC;
Mel Gorman11e33f62009-06-16 15:31:57 -07002232 lockdep_set_current_reclaim_state(gfp_mask);
2233 reclaim_state.reclaimed_slab = 0;
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08002234 current->reclaim_state = &reclaim_state;
Mel Gorman11e33f62009-06-16 15:31:57 -07002235
Marek Szyprowski1ed36512012-01-25 12:09:52 +01002236 progress = try_to_free_pages(zonelist, order, gfp_mask, nodemask);
Mel Gorman11e33f62009-06-16 15:31:57 -07002237
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08002238 current->reclaim_state = NULL;
Mel Gorman11e33f62009-06-16 15:31:57 -07002239 lockdep_clear_current_reclaim_state();
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08002240 current->flags &= ~PF_MEMALLOC;
Mel Gorman11e33f62009-06-16 15:31:57 -07002241
2242 cond_resched();
2243
Marek Szyprowski1ed36512012-01-25 12:09:52 +01002244 return progress;
2245}
2246
2247/* The really slow allocator path where we enter direct reclaim */
2248static inline struct page *
2249__alloc_pages_direct_reclaim(gfp_t gfp_mask, unsigned int order,
2250 struct zonelist *zonelist, enum zone_type high_zoneidx,
2251 nodemask_t *nodemask, int alloc_flags, struct zone *preferred_zone,
2252 int migratetype, unsigned long *did_some_progress)
2253{
2254 struct page *page = NULL;
2255 bool drained = false;
2256
2257 *did_some_progress = __perform_reclaim(gfp_mask, order, zonelist,
2258 nodemask);
Mel Gorman9ee493c2010-09-09 16:38:18 -07002259 if (unlikely(!(*did_some_progress)))
2260 return NULL;
Mel Gorman11e33f62009-06-16 15:31:57 -07002261
Mel Gorman76d3fbf2011-07-25 17:12:30 -07002262 /* After successful reclaim, reconsider all zones for allocation */
2263 if (NUMA_BUILD)
2264 zlc_clear_zones_full(zonelist);
2265
Mel Gorman9ee493c2010-09-09 16:38:18 -07002266retry:
2267 page = get_page_from_freelist(gfp_mask, nodemask, order,
Mel Gorman5117f452009-06-16 15:31:59 -07002268 zonelist, high_zoneidx,
Mel Gorman3dd28262009-06-16 15:32:00 -07002269 alloc_flags, preferred_zone,
2270 migratetype);
Mel Gorman9ee493c2010-09-09 16:38:18 -07002271
2272 /*
2273 * If an allocation failed after direct reclaim, it could be because
2274 * pages are pinned on the per-cpu lists. Drain them and try again
2275 */
2276 if (!page && !drained) {
2277 drain_all_pages();
2278 drained = true;
2279 goto retry;
2280 }
2281
Mel Gorman11e33f62009-06-16 15:31:57 -07002282 return page;
2283}
2284
Mel Gorman11e33f62009-06-16 15:31:57 -07002285/*
2286 * This is called in the allocator slow-path if the allocation request is of
2287 * sufficient urgency to ignore watermarks and take other desperate measures
2288 */
2289static inline struct page *
2290__alloc_pages_high_priority(gfp_t gfp_mask, unsigned int order,
2291 struct zonelist *zonelist, enum zone_type high_zoneidx,
Mel Gorman3dd28262009-06-16 15:32:00 -07002292 nodemask_t *nodemask, struct zone *preferred_zone,
2293 int migratetype)
Mel Gorman11e33f62009-06-16 15:31:57 -07002294{
2295 struct page *page;
2296
2297 do {
2298 page = get_page_from_freelist(gfp_mask, nodemask, order,
Mel Gorman5117f452009-06-16 15:31:59 -07002299 zonelist, high_zoneidx, ALLOC_NO_WATERMARKS,
Mel Gorman3dd28262009-06-16 15:32:00 -07002300 preferred_zone, migratetype);
Mel Gorman11e33f62009-06-16 15:31:57 -07002301
2302 if (!page && gfp_mask & __GFP_NOFAIL)
Mel Gorman0e093d992010-10-26 14:21:45 -07002303 wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50);
Mel Gorman11e33f62009-06-16 15:31:57 -07002304 } while (!page && (gfp_mask & __GFP_NOFAIL));
2305
2306 return page;
2307}
2308
2309static inline
2310void wake_all_kswapd(unsigned int order, struct zonelist *zonelist,
Mel Gorman99504742011-01-13 15:46:20 -08002311 enum zone_type high_zoneidx,
2312 enum zone_type classzone_idx)
Mel Gorman11e33f62009-06-16 15:31:57 -07002313{
2314 struct zoneref *z;
2315 struct zone *zone;
2316
2317 for_each_zone_zonelist(zone, z, zonelist, high_zoneidx)
Mel Gorman99504742011-01-13 15:46:20 -08002318 wakeup_kswapd(zone, order, classzone_idx);
Mel Gorman11e33f62009-06-16 15:31:57 -07002319}
2320
Peter Zijlstra341ce062009-06-16 15:32:02 -07002321static inline int
2322gfp_to_alloc_flags(gfp_t gfp_mask)
2323{
Peter Zijlstra341ce062009-06-16 15:32:02 -07002324 int alloc_flags = ALLOC_WMARK_MIN | ALLOC_CPUSET;
2325 const gfp_t wait = gfp_mask & __GFP_WAIT;
2326
Mel Gormana56f57f2009-06-16 15:32:02 -07002327 /* __GFP_HIGH is assumed to be the same as ALLOC_HIGH to save a branch. */
Namhyung Kime6223a32010-10-26 14:21:59 -07002328 BUILD_BUG_ON(__GFP_HIGH != (__force gfp_t) ALLOC_HIGH);
Mel Gormana56f57f2009-06-16 15:32:02 -07002329
Peter Zijlstra341ce062009-06-16 15:32:02 -07002330 /*
2331 * The caller may dip into page reserves a bit more if the caller
2332 * cannot run direct reclaim, or if the caller has realtime scheduling
2333 * policy or is asking for __GFP_HIGH memory. GFP_ATOMIC requests will
2334 * set both ALLOC_HARDER (!wait) and ALLOC_HIGH (__GFP_HIGH).
2335 */
Namhyung Kime6223a32010-10-26 14:21:59 -07002336 alloc_flags |= (__force int) (gfp_mask & __GFP_HIGH);
Peter Zijlstra341ce062009-06-16 15:32:02 -07002337
2338 if (!wait) {
Andrea Arcangeli5c3240d2011-01-13 15:46:49 -08002339 /*
2340 * Not worth trying to allocate harder for
2341 * __GFP_NOMEMALLOC even if it can't schedule.
2342 */
2343 if (!(gfp_mask & __GFP_NOMEMALLOC))
2344 alloc_flags |= ALLOC_HARDER;
Peter Zijlstra341ce062009-06-16 15:32:02 -07002345 /*
2346 * Ignore cpuset if GFP_ATOMIC (!wait) rather than fail alloc.
2347 * See also cpuset_zone_allowed() comment in kernel/cpuset.c.
2348 */
2349 alloc_flags &= ~ALLOC_CPUSET;
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08002350 } else if (unlikely(rt_task(current)) && !in_interrupt())
Peter Zijlstra341ce062009-06-16 15:32:02 -07002351 alloc_flags |= ALLOC_HARDER;
2352
2353 if (likely(!(gfp_mask & __GFP_NOMEMALLOC))) {
2354 if (!in_interrupt() &&
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08002355 ((current->flags & PF_MEMALLOC) ||
Peter Zijlstra341ce062009-06-16 15:32:02 -07002356 unlikely(test_thread_flag(TIF_MEMDIE))))
2357 alloc_flags |= ALLOC_NO_WATERMARKS;
2358 }
Bartlomiej Zolnierkiewicz19644312012-10-08 16:32:05 -07002359#ifdef CONFIG_CMA
2360 if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
2361 alloc_flags |= ALLOC_CMA;
2362#endif
Peter Zijlstra341ce062009-06-16 15:32:02 -07002363 return alloc_flags;
2364}
2365
Mel Gorman11e33f62009-06-16 15:31:57 -07002366static inline struct page *
2367__alloc_pages_slowpath(gfp_t gfp_mask, unsigned int order,
2368 struct zonelist *zonelist, enum zone_type high_zoneidx,
Mel Gorman3dd28262009-06-16 15:32:00 -07002369 nodemask_t *nodemask, struct zone *preferred_zone,
2370 int migratetype)
Mel Gorman11e33f62009-06-16 15:31:57 -07002371{
2372 const gfp_t wait = gfp_mask & __GFP_WAIT;
2373 struct page *page = NULL;
2374 int alloc_flags;
2375 unsigned long pages_reclaimed = 0;
2376 unsigned long did_some_progress;
Mel Gorman77f1fe62011-01-13 15:45:57 -08002377 bool sync_migration = false;
Mel Gorman66199712012-01-12 17:19:41 -08002378 bool deferred_compaction = false;
Mel Gorman27d1af82012-08-21 16:16:17 -07002379 bool contended_compaction = false;
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08002380
Christoph Lameter952f3b52006-12-06 20:33:26 -08002381 /*
Mel Gorman72807a72009-06-16 15:32:18 -07002382 * In the slowpath, we sanity check order to avoid ever trying to
2383 * reclaim >= MAX_ORDER areas which will never succeed. Callers may
2384 * be using allocators in order of preference for an area that is
2385 * too large.
2386 */
Mel Gorman1fc28b72009-07-29 15:04:08 -07002387 if (order >= MAX_ORDER) {
2388 WARN_ON_ONCE(!(gfp_mask & __GFP_NOWARN));
Mel Gorman72807a72009-06-16 15:32:18 -07002389 return NULL;
Mel Gorman1fc28b72009-07-29 15:04:08 -07002390 }
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08002391
Christoph Lameter952f3b52006-12-06 20:33:26 -08002392 /*
2393 * GFP_THISNODE (meaning __GFP_THISNODE, __GFP_NORETRY and
2394 * __GFP_NOWARN set) should not cause reclaim since the subsystem
2395 * (f.e. slab) using GFP_THISNODE may choose to trigger reclaim
2396 * using a larger set of nodes after it has established that the
2397 * allowed per node queues are empty and that nodes are
2398 * over allocated.
2399 */
2400 if (NUMA_BUILD && (gfp_mask & GFP_THISNODE) == GFP_THISNODE)
2401 goto nopage;
2402
Mel Gormancc4a6852009-11-11 14:26:14 -08002403restart:
Andrea Arcangeli32dba982011-01-13 15:46:49 -08002404 if (!(gfp_mask & __GFP_NO_KSWAPD))
2405 wake_all_kswapd(order, zonelist, high_zoneidx,
Mel Gorman99504742011-01-13 15:46:20 -08002406 zone_idx(preferred_zone));
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08002407
Paul Jackson9bf22292005-09-06 15:18:12 -07002408 /*
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08002409 * OK, we're below the kswapd watermark and have kicked background
2410 * reclaim. Now things get more complex, so set up alloc_flags according
2411 * to how we want to proceed.
Paul Jackson9bf22292005-09-06 15:18:12 -07002412 */
Peter Zijlstra341ce062009-06-16 15:32:02 -07002413 alloc_flags = gfp_to_alloc_flags(gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002414
David Rientjesf33261d2011-01-25 15:07:20 -08002415 /*
2416 * Find the true preferred zone if the allocation is unconstrained by
2417 * cpusets.
2418 */
2419 if (!(alloc_flags & ALLOC_CPUSET) && !nodemask)
2420 first_zones_zonelist(zonelist, high_zoneidx, NULL,
2421 &preferred_zone);
2422
Andrew Barrycfa54a02011-05-24 17:12:52 -07002423rebalance:
Peter Zijlstra341ce062009-06-16 15:32:02 -07002424 /* This is the last chance, in general, before the goto nopage. */
Mel Gorman19770b32008-04-28 02:12:18 -07002425 page = get_page_from_freelist(gfp_mask, nodemask, order, zonelist,
Peter Zijlstra341ce062009-06-16 15:32:02 -07002426 high_zoneidx, alloc_flags & ~ALLOC_NO_WATERMARKS,
2427 preferred_zone, migratetype);
Rohit Seth7fb1d9f2005-11-13 16:06:43 -08002428 if (page)
2429 goto got_pg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002430
Mel Gorman11e33f62009-06-16 15:31:57 -07002431 /* Allocate without watermarks if the context allows */
Peter Zijlstra341ce062009-06-16 15:32:02 -07002432 if (alloc_flags & ALLOC_NO_WATERMARKS) {
2433 page = __alloc_pages_high_priority(gfp_mask, order,
2434 zonelist, high_zoneidx, nodemask,
2435 preferred_zone, migratetype);
2436 if (page)
2437 goto got_pg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002438 }
2439
2440 /* Atomic allocations - we can't balance anything */
2441 if (!wait)
2442 goto nopage;
2443
Peter Zijlstra341ce062009-06-16 15:32:02 -07002444 /* Avoid recursion of direct reclaim */
Andrew Mortonc06b1fc2011-01-13 15:47:32 -08002445 if (current->flags & PF_MEMALLOC)
Peter Zijlstra341ce062009-06-16 15:32:02 -07002446 goto nopage;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002447
David Rientjes6583bb62009-07-29 15:02:06 -07002448 /* Avoid allocations with no watermarks from looping endlessly */
2449 if (test_thread_flag(TIF_MEMDIE) && !(gfp_mask & __GFP_NOFAIL))
2450 goto nopage;
2451
Mel Gorman77f1fe62011-01-13 15:45:57 -08002452 /*
2453 * Try direct compaction. The first pass is asynchronous. Subsequent
2454 * attempts after direct reclaim are synchronous
2455 */
Mel Gorman56de7262010-05-24 14:32:30 -07002456 page = __alloc_pages_direct_compact(gfp_mask, order,
2457 zonelist, high_zoneidx,
2458 nodemask,
2459 alloc_flags, preferred_zone,
Mel Gorman66199712012-01-12 17:19:41 -08002460 migratetype, sync_migration,
Mel Gorman27d1af82012-08-21 16:16:17 -07002461 &contended_compaction,
Mel Gorman66199712012-01-12 17:19:41 -08002462 &deferred_compaction,
2463 &did_some_progress);
Mel Gorman56de7262010-05-24 14:32:30 -07002464 if (page)
2465 goto got_pg;
Andrea Arcangelic6a140b2011-05-24 17:11:38 -07002466 sync_migration = true;
Mel Gorman56de7262010-05-24 14:32:30 -07002467
Mel Gorman66199712012-01-12 17:19:41 -08002468 /*
2469 * If compaction is deferred for high-order allocations, it is because
2470 * sync compaction recently failed. In this is the case and the caller
Mel Gorman27d1af82012-08-21 16:16:17 -07002471 * requested a movable allocation that does not heavily disrupt the
2472 * system then fail the allocation instead of entering direct reclaim.
Mel Gorman66199712012-01-12 17:19:41 -08002473 */
Mel Gorman27d1af82012-08-21 16:16:17 -07002474 if ((deferred_compaction || contended_compaction) &&
2475 (gfp_mask & __GFP_NO_KSWAPD))
Mel Gorman66199712012-01-12 17:19:41 -08002476 goto nopage;
2477
Mel Gorman11e33f62009-06-16 15:31:57 -07002478 /* Try direct reclaim and then allocating */
2479 page = __alloc_pages_direct_reclaim(gfp_mask, order,
2480 zonelist, high_zoneidx,
2481 nodemask,
Mel Gorman5117f452009-06-16 15:31:59 -07002482 alloc_flags, preferred_zone,
Mel Gorman3dd28262009-06-16 15:32:00 -07002483 migratetype, &did_some_progress);
Mel Gorman11e33f62009-06-16 15:31:57 -07002484 if (page)
2485 goto got_pg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002486
Linus Torvalds1da177e2005-04-16 15:20:36 -07002487 /*
Mel Gorman11e33f62009-06-16 15:31:57 -07002488 * If we failed to make any progress reclaiming, then we are
2489 * running out of options and have to consider going OOM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002490 */
Mel Gorman11e33f62009-06-16 15:31:57 -07002491 if (!did_some_progress) {
2492 if ((gfp_mask & __GFP_FS) && !(gfp_mask & __GFP_NORETRY)) {
Rafael J. Wysocki7f33d492009-06-16 15:32:41 -07002493 if (oom_killer_disabled)
2494 goto nopage;
David Rientjes29fd66d2012-03-28 14:42:41 -07002495 /* Coredumps can quickly deplete all memory reserves */
2496 if ((current->flags & PF_DUMPCORE) &&
2497 !(gfp_mask & __GFP_NOFAIL))
2498 goto nopage;
Mel Gorman11e33f62009-06-16 15:31:57 -07002499 page = __alloc_pages_may_oom(gfp_mask, order,
2500 zonelist, high_zoneidx,
Mel Gorman3dd28262009-06-16 15:32:00 -07002501 nodemask, preferred_zone,
2502 migratetype);
Mel Gorman11e33f62009-06-16 15:31:57 -07002503 if (page)
2504 goto got_pg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002505
David Rientjes03668b32010-08-09 17:18:54 -07002506 if (!(gfp_mask & __GFP_NOFAIL)) {
2507 /*
2508 * The oom killer is not called for high-order
2509 * allocations that may fail, so if no progress
2510 * is being made, there are no other options and
2511 * retrying is unlikely to help.
2512 */
2513 if (order > PAGE_ALLOC_COSTLY_ORDER)
2514 goto nopage;
2515 /*
2516 * The oom killer is not called for lowmem
2517 * allocations to prevent needlessly killing
2518 * innocent tasks.
2519 */
2520 if (high_zoneidx < ZONE_NORMAL)
2521 goto nopage;
2522 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07002523
Linus Torvalds1da177e2005-04-16 15:20:36 -07002524 goto restart;
2525 }
2526 }
2527
Mel Gorman11e33f62009-06-16 15:31:57 -07002528 /* Check if we should retry the allocation */
Nishanth Aravamudana41f24e2008-04-29 00:58:25 -07002529 pages_reclaimed += did_some_progress;
Mel Gormanf90ac392012-01-10 15:07:15 -08002530 if (should_alloc_retry(gfp_mask, order, did_some_progress,
2531 pages_reclaimed)) {
Mel Gorman11e33f62009-06-16 15:31:57 -07002532 /* Wait for some write requests to complete then retry */
Mel Gorman0e093d992010-10-26 14:21:45 -07002533 wait_iff_congested(preferred_zone, BLK_RW_ASYNC, HZ/50);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002534 goto rebalance;
Mel Gorman3e7d3442011-01-13 15:45:56 -08002535 } else {
2536 /*
2537 * High-order allocations do not necessarily loop after
2538 * direct reclaim and reclaim/compaction depends on compaction
2539 * being called after reclaim so call directly if necessary
2540 */
2541 page = __alloc_pages_direct_compact(gfp_mask, order,
2542 zonelist, high_zoneidx,
2543 nodemask,
2544 alloc_flags, preferred_zone,
Mel Gorman66199712012-01-12 17:19:41 -08002545 migratetype, sync_migration,
Mel Gorman27d1af82012-08-21 16:16:17 -07002546 &contended_compaction,
Mel Gorman66199712012-01-12 17:19:41 -08002547 &deferred_compaction,
2548 &did_some_progress);
Mel Gorman3e7d3442011-01-13 15:45:56 -08002549 if (page)
2550 goto got_pg;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002551 }
2552
2553nopage:
Dave Hansena238ab52011-05-24 17:12:16 -07002554 warn_alloc_failed(gfp_mask, order, NULL);
Vegard Nossumb1eeab62008-11-25 16:55:53 +01002555 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002556got_pg:
Vegard Nossumb1eeab62008-11-25 16:55:53 +01002557 if (kmemcheck_enabled)
2558 kmemcheck_pagealloc_alloc(page, order, gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002559 return page;
Mel Gorman11e33f62009-06-16 15:31:57 -07002560
Linus Torvalds1da177e2005-04-16 15:20:36 -07002561}
Mel Gorman11e33f62009-06-16 15:31:57 -07002562
2563/*
2564 * This is the 'heart' of the zoned buddy allocator.
2565 */
2566struct page *
2567__alloc_pages_nodemask(gfp_t gfp_mask, unsigned int order,
2568 struct zonelist *zonelist, nodemask_t *nodemask)
2569{
2570 enum zone_type high_zoneidx = gfp_zone(gfp_mask);
Mel Gorman5117f452009-06-16 15:31:59 -07002571 struct zone *preferred_zone;
Mel Gormancc9a6c82012-03-21 16:34:11 -07002572 struct page *page = NULL;
Mel Gorman3dd28262009-06-16 15:32:00 -07002573 int migratetype = allocflags_to_migratetype(gfp_mask);
Mel Gormancc9a6c82012-03-21 16:34:11 -07002574 unsigned int cpuset_mems_cookie;
Bartlomiej Zolnierkiewicz19644312012-10-08 16:32:05 -07002575 int alloc_flags = ALLOC_WMARK_LOW|ALLOC_CPUSET;
Mel Gorman11e33f62009-06-16 15:31:57 -07002576
Benjamin Herrenschmidtdcce2842009-06-18 13:24:12 +10002577 gfp_mask &= gfp_allowed_mask;
2578
Mel Gorman11e33f62009-06-16 15:31:57 -07002579 lockdep_trace_alloc(gfp_mask);
2580
2581 might_sleep_if(gfp_mask & __GFP_WAIT);
2582
2583 if (should_fail_alloc_page(gfp_mask, order))
2584 return NULL;
2585
2586 /*
2587 * Check the zones suitable for the gfp_mask contain at least one
2588 * valid zone. It's possible to have an empty zonelist as a result
2589 * of GFP_THISNODE and a memoryless node
2590 */
2591 if (unlikely(!zonelist->_zonerefs->zone))
2592 return NULL;
2593
Mel Gormancc9a6c82012-03-21 16:34:11 -07002594retry_cpuset:
2595 cpuset_mems_cookie = get_mems_allowed();
2596
Mel Gorman5117f452009-06-16 15:31:59 -07002597 /* The preferred zone is used for statistics later */
David Rientjesf33261d2011-01-25 15:07:20 -08002598 first_zones_zonelist(zonelist, high_zoneidx,
2599 nodemask ? : &cpuset_current_mems_allowed,
2600 &preferred_zone);
Mel Gormancc9a6c82012-03-21 16:34:11 -07002601 if (!preferred_zone)
2602 goto out;
Mel Gorman5117f452009-06-16 15:31:59 -07002603
Bartlomiej Zolnierkiewicz19644312012-10-08 16:32:05 -07002604#ifdef CONFIG_CMA
2605 if (allocflags_to_migratetype(gfp_mask) == MIGRATE_MOVABLE)
2606 alloc_flags |= ALLOC_CMA;
2607#endif
Mel Gorman5117f452009-06-16 15:31:59 -07002608 /* First allocation attempt */
Mel Gorman11e33f62009-06-16 15:31:57 -07002609 page = get_page_from_freelist(gfp_mask|__GFP_HARDWALL, nodemask, order,
Bartlomiej Zolnierkiewicz19644312012-10-08 16:32:05 -07002610 zonelist, high_zoneidx, alloc_flags,
Mel Gorman3dd28262009-06-16 15:32:00 -07002611 preferred_zone, migratetype);
Mel Gorman11e33f62009-06-16 15:31:57 -07002612 if (unlikely(!page))
2613 page = __alloc_pages_slowpath(gfp_mask, order,
Mel Gorman5117f452009-06-16 15:31:59 -07002614 zonelist, high_zoneidx, nodemask,
Mel Gorman3dd28262009-06-16 15:32:00 -07002615 preferred_zone, migratetype);
Mel Gorman11e33f62009-06-16 15:31:57 -07002616
Mel Gorman4b4f2782009-09-21 17:02:41 -07002617 trace_mm_page_alloc(page, order, gfp_mask, migratetype);
Mel Gormancc9a6c82012-03-21 16:34:11 -07002618
2619out:
2620 /*
2621 * When updating a task's mems_allowed, it is possible to race with
2622 * parallel threads in such a way that an allocation can fail while
2623 * the mask is being updated. If a page allocation is about to fail,
2624 * check if the cpuset changed during allocation and if so, retry.
2625 */
2626 if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
2627 goto retry_cpuset;
2628
Mel Gorman11e33f62009-06-16 15:31:57 -07002629 return page;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002630}
Mel Gormand2391712009-06-16 15:31:52 -07002631EXPORT_SYMBOL(__alloc_pages_nodemask);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002632
2633/*
2634 * Common helper functions.
2635 */
Harvey Harrison920c7a52008-02-04 22:29:26 -08002636unsigned long __get_free_pages(gfp_t gfp_mask, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002637{
Akinobu Mita945a1112009-09-21 17:01:47 -07002638 struct page *page;
2639
2640 /*
2641 * __get_free_pages() returns a 32-bit address, which cannot represent
2642 * a highmem page
2643 */
2644 VM_BUG_ON((gfp_mask & __GFP_HIGHMEM) != 0);
2645
Linus Torvalds1da177e2005-04-16 15:20:36 -07002646 page = alloc_pages(gfp_mask, order);
2647 if (!page)
2648 return 0;
2649 return (unsigned long) page_address(page);
2650}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002651EXPORT_SYMBOL(__get_free_pages);
2652
Harvey Harrison920c7a52008-02-04 22:29:26 -08002653unsigned long get_zeroed_page(gfp_t gfp_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002654{
Akinobu Mita945a1112009-09-21 17:01:47 -07002655 return __get_free_pages(gfp_mask | __GFP_ZERO, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002656}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002657EXPORT_SYMBOL(get_zeroed_page);
2658
Harvey Harrison920c7a52008-02-04 22:29:26 -08002659void __free_pages(struct page *page, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002660{
Nick Pigginb5810032005-10-29 18:16:12 -07002661 if (put_page_testzero(page)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002662 if (order == 0)
Li Hongfc916682010-03-05 13:41:54 -08002663 free_hot_cold_page(page, 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002664 else
2665 __free_pages_ok(page, order);
2666 }
2667}
2668
2669EXPORT_SYMBOL(__free_pages);
2670
Harvey Harrison920c7a52008-02-04 22:29:26 -08002671void free_pages(unsigned long addr, unsigned int order)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002672{
2673 if (addr != 0) {
Nick Piggin725d7042006-09-25 23:30:55 -07002674 VM_BUG_ON(!virt_addr_valid((void *)addr));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002675 __free_pages(virt_to_page((void *)addr), order);
2676 }
2677}
2678
2679EXPORT_SYMBOL(free_pages);
2680
Andi Kleenee85c2e2011-05-11 15:13:34 -07002681static void *make_alloc_exact(unsigned long addr, unsigned order, size_t size)
2682{
2683 if (addr) {
2684 unsigned long alloc_end = addr + (PAGE_SIZE << order);
2685 unsigned long used = addr + PAGE_ALIGN(size);
2686
2687 split_page(virt_to_page((void *)addr), order);
2688 while (used < alloc_end) {
2689 free_page(used);
2690 used += PAGE_SIZE;
2691 }
2692 }
2693 return (void *)addr;
2694}
2695
Timur Tabi2be0ffe2008-07-23 21:28:11 -07002696/**
2697 * alloc_pages_exact - allocate an exact number physically-contiguous pages.
2698 * @size: the number of bytes to allocate
2699 * @gfp_mask: GFP flags for the allocation
2700 *
2701 * This function is similar to alloc_pages(), except that it allocates the
2702 * minimum number of pages to satisfy the request. alloc_pages() can only
2703 * allocate memory in power-of-two pages.
2704 *
2705 * This function is also limited by MAX_ORDER.
2706 *
2707 * Memory allocated by this function must be released by free_pages_exact().
2708 */
2709void *alloc_pages_exact(size_t size, gfp_t gfp_mask)
2710{
2711 unsigned int order = get_order(size);
2712 unsigned long addr;
2713
2714 addr = __get_free_pages(gfp_mask, order);
Andi Kleenee85c2e2011-05-11 15:13:34 -07002715 return make_alloc_exact(addr, order, size);
Timur Tabi2be0ffe2008-07-23 21:28:11 -07002716}
2717EXPORT_SYMBOL(alloc_pages_exact);
2718
2719/**
Andi Kleenee85c2e2011-05-11 15:13:34 -07002720 * alloc_pages_exact_nid - allocate an exact number of physically-contiguous
2721 * pages on a node.
Randy Dunlapb5e6ab52011-05-16 13:16:54 -07002722 * @nid: the preferred node ID where memory should be allocated
Andi Kleenee85c2e2011-05-11 15:13:34 -07002723 * @size: the number of bytes to allocate
2724 * @gfp_mask: GFP flags for the allocation
2725 *
2726 * Like alloc_pages_exact(), but try to allocate on node nid first before falling
2727 * back.
2728 * Note this is not alloc_pages_exact_node() which allocates on a specific node,
2729 * but is not exact.
2730 */
2731void *alloc_pages_exact_nid(int nid, size_t size, gfp_t gfp_mask)
2732{
2733 unsigned order = get_order(size);
2734 struct page *p = alloc_pages_node(nid, gfp_mask, order);
2735 if (!p)
2736 return NULL;
2737 return make_alloc_exact((unsigned long)page_address(p), order, size);
2738}
2739EXPORT_SYMBOL(alloc_pages_exact_nid);
2740
2741/**
Timur Tabi2be0ffe2008-07-23 21:28:11 -07002742 * free_pages_exact - release memory allocated via alloc_pages_exact()
2743 * @virt: the value returned by alloc_pages_exact.
2744 * @size: size of allocation, same value as passed to alloc_pages_exact().
2745 *
2746 * Release the memory allocated by a previous call to alloc_pages_exact.
2747 */
2748void free_pages_exact(void *virt, size_t size)
2749{
2750 unsigned long addr = (unsigned long)virt;
2751 unsigned long end = addr + PAGE_ALIGN(size);
2752
2753 while (addr < end) {
2754 free_page(addr);
2755 addr += PAGE_SIZE;
2756 }
2757}
2758EXPORT_SYMBOL(free_pages_exact);
2759
Linus Torvalds1da177e2005-04-16 15:20:36 -07002760static unsigned int nr_free_zone_pages(int offset)
2761{
Mel Gormandd1a2392008-04-28 02:12:17 -07002762 struct zoneref *z;
Mel Gorman54a6eb52008-04-28 02:12:16 -07002763 struct zone *zone;
2764
Martin J. Blighe310fd42005-07-29 22:59:18 -07002765 /* Just pick one node, since fallback list is circular */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002766 unsigned int sum = 0;
2767
Mel Gorman0e884602008-04-28 02:12:14 -07002768 struct zonelist *zonelist = node_zonelist(numa_node_id(), GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002769
Mel Gorman54a6eb52008-04-28 02:12:16 -07002770 for_each_zone_zonelist(zone, z, zonelist, offset) {
Martin J. Blighe310fd42005-07-29 22:59:18 -07002771 unsigned long size = zone->present_pages;
Mel Gorman41858962009-06-16 15:32:12 -07002772 unsigned long high = high_wmark_pages(zone);
Martin J. Blighe310fd42005-07-29 22:59:18 -07002773 if (size > high)
2774 sum += size - high;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002775 }
2776
2777 return sum;
2778}
2779
2780/*
2781 * Amount of free RAM allocatable within ZONE_DMA and ZONE_NORMAL
2782 */
2783unsigned int nr_free_buffer_pages(void)
2784{
Al Viroaf4ca452005-10-21 02:55:38 -04002785 return nr_free_zone_pages(gfp_zone(GFP_USER));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002786}
Meelap Shahc2f1a552007-07-17 04:04:39 -07002787EXPORT_SYMBOL_GPL(nr_free_buffer_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002788
2789/*
2790 * Amount of free RAM allocatable within all zones
2791 */
2792unsigned int nr_free_pagecache_pages(void)
2793{
Mel Gorman2a1e2742007-07-17 04:03:12 -07002794 return nr_free_zone_pages(gfp_zone(GFP_HIGHUSER_MOVABLE));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002795}
Christoph Lameter08e0f6a2006-09-27 01:50:06 -07002796
2797static inline void show_node(struct zone *zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002798{
Christoph Lameter08e0f6a2006-09-27 01:50:06 -07002799 if (NUMA_BUILD)
Andy Whitcroft25ba77c2006-12-06 20:33:03 -08002800 printk("Node %d ", zone_to_nid(zone));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002801}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002802
Linus Torvalds1da177e2005-04-16 15:20:36 -07002803void si_meminfo(struct sysinfo *val)
2804{
2805 val->totalram = totalram_pages;
2806 val->sharedram = 0;
Christoph Lameterd23ad422007-02-10 01:43:02 -08002807 val->freeram = global_page_state(NR_FREE_PAGES);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002808 val->bufferram = nr_blockdev_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002809 val->totalhigh = totalhigh_pages;
2810 val->freehigh = nr_free_highpages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07002811 val->mem_unit = PAGE_SIZE;
2812}
2813
2814EXPORT_SYMBOL(si_meminfo);
2815
2816#ifdef CONFIG_NUMA
2817void si_meminfo_node(struct sysinfo *val, int nid)
2818{
2819 pg_data_t *pgdat = NODE_DATA(nid);
2820
2821 val->totalram = pgdat->node_present_pages;
Christoph Lameterd23ad422007-02-10 01:43:02 -08002822 val->freeram = node_page_state(nid, NR_FREE_PAGES);
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07002823#ifdef CONFIG_HIGHMEM
Linus Torvalds1da177e2005-04-16 15:20:36 -07002824 val->totalhigh = pgdat->node_zones[ZONE_HIGHMEM].present_pages;
Christoph Lameterd23ad422007-02-10 01:43:02 -08002825 val->freehigh = zone_page_state(&pgdat->node_zones[ZONE_HIGHMEM],
2826 NR_FREE_PAGES);
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07002827#else
2828 val->totalhigh = 0;
2829 val->freehigh = 0;
2830#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07002831 val->mem_unit = PAGE_SIZE;
2832}
2833#endif
2834
David Rientjesddd588b2011-03-22 16:30:46 -07002835/*
David Rientjes7bf02ea2011-05-24 17:11:16 -07002836 * Determine whether the node should be displayed or not, depending on whether
2837 * SHOW_MEM_FILTER_NODES was passed to show_free_areas().
David Rientjesddd588b2011-03-22 16:30:46 -07002838 */
David Rientjes7bf02ea2011-05-24 17:11:16 -07002839bool skip_free_areas_node(unsigned int flags, int nid)
David Rientjesddd588b2011-03-22 16:30:46 -07002840{
2841 bool ret = false;
Mel Gormancc9a6c82012-03-21 16:34:11 -07002842 unsigned int cpuset_mems_cookie;
David Rientjesddd588b2011-03-22 16:30:46 -07002843
2844 if (!(flags & SHOW_MEM_FILTER_NODES))
2845 goto out;
2846
Mel Gormancc9a6c82012-03-21 16:34:11 -07002847 do {
2848 cpuset_mems_cookie = get_mems_allowed();
2849 ret = !node_isset(nid, cpuset_current_mems_allowed);
2850 } while (!put_mems_allowed(cpuset_mems_cookie));
David Rientjesddd588b2011-03-22 16:30:46 -07002851out:
2852 return ret;
2853}
2854
Linus Torvalds1da177e2005-04-16 15:20:36 -07002855#define K(x) ((x) << (PAGE_SHIFT-10))
2856
Rabin Vincent394b8082012-12-11 16:00:24 -08002857static void show_migration_types(unsigned char type)
2858{
2859 static const char types[MIGRATE_TYPES] = {
2860 [MIGRATE_UNMOVABLE] = 'U',
2861 [MIGRATE_RECLAIMABLE] = 'E',
2862 [MIGRATE_MOVABLE] = 'M',
2863 [MIGRATE_RESERVE] = 'R',
2864#ifdef CONFIG_CMA
2865 [MIGRATE_CMA] = 'C',
2866#endif
2867 [MIGRATE_ISOLATE] = 'I',
2868 };
2869 char tmp[MIGRATE_TYPES + 1];
2870 char *p = tmp;
2871 int i;
2872
2873 for (i = 0; i < MIGRATE_TYPES; i++) {
2874 if (type & (1 << i))
2875 *p++ = types[i];
2876 }
2877
2878 *p = '\0';
2879 printk("(%s) ", tmp);
2880}
2881
Linus Torvalds1da177e2005-04-16 15:20:36 -07002882/*
2883 * Show free area list (used inside shift_scroll-lock stuff)
2884 * We also calculate the percentage fragmentation. We do this by counting the
2885 * memory on each free list with the exception of the first item on the list.
David Rientjesddd588b2011-03-22 16:30:46 -07002886 * Suppresses nodes that are not allowed by current's cpuset if
2887 * SHOW_MEM_FILTER_NODES is passed.
Linus Torvalds1da177e2005-04-16 15:20:36 -07002888 */
David Rientjes7bf02ea2011-05-24 17:11:16 -07002889void show_free_areas(unsigned int filter)
Linus Torvalds1da177e2005-04-16 15:20:36 -07002890{
Jes Sorensenc7241912006-09-27 01:50:05 -07002891 int cpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002892 struct zone *zone;
2893
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07002894 for_each_populated_zone(zone) {
David Rientjes7bf02ea2011-05-24 17:11:16 -07002895 if (skip_free_areas_node(filter, zone_to_nid(zone)))
David Rientjesddd588b2011-03-22 16:30:46 -07002896 continue;
Jes Sorensenc7241912006-09-27 01:50:05 -07002897 show_node(zone);
2898 printk("%s per-cpu:\n", zone->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002899
Dave Jones6b482c62005-11-10 15:45:56 -05002900 for_each_online_cpu(cpu) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002901 struct per_cpu_pageset *pageset;
2902
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09002903 pageset = per_cpu_ptr(zone->pageset, cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002904
Christoph Lameter3dfa5722008-02-04 22:29:19 -08002905 printk("CPU %4d: hi:%5d, btch:%4d usd:%4d\n",
2906 cpu, pageset->pcp.high,
2907 pageset->pcp.batch, pageset->pcp.count);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002908 }
2909 }
2910
KOSAKI Motohiroa7312862009-09-21 17:01:37 -07002911 printk("active_anon:%lu inactive_anon:%lu isolated_anon:%lu\n"
2912 " active_file:%lu inactive_file:%lu isolated_file:%lu\n"
Lee Schermerhorn7b854122008-10-18 20:26:40 -07002913 " unevictable:%lu"
Andrew Mortonb76146e2009-10-26 16:49:52 -07002914 " dirty:%lu writeback:%lu unstable:%lu\n"
KOSAKI Motohiro3701b032009-09-21 17:01:29 -07002915 " free:%lu slab_reclaimable:%lu slab_unreclaimable:%lu\n"
Larry Bassel6d6e2c92012-12-14 14:21:05 -08002916 " mapped:%lu shmem:%lu pagetables:%lu bounce:%lu\n"
2917 " free_cma:%lu\n",
Rik van Riel4f98a2f2008-10-18 20:26:32 -07002918 global_page_state(NR_ACTIVE_ANON),
Rik van Riel4f98a2f2008-10-18 20:26:32 -07002919 global_page_state(NR_INACTIVE_ANON),
KOSAKI Motohiroa7312862009-09-21 17:01:37 -07002920 global_page_state(NR_ISOLATED_ANON),
2921 global_page_state(NR_ACTIVE_FILE),
Rik van Riel4f98a2f2008-10-18 20:26:32 -07002922 global_page_state(NR_INACTIVE_FILE),
KOSAKI Motohiroa7312862009-09-21 17:01:37 -07002923 global_page_state(NR_ISOLATED_FILE),
Lee Schermerhorn7b854122008-10-18 20:26:40 -07002924 global_page_state(NR_UNEVICTABLE),
Christoph Lameterb1e7a8f2006-06-30 01:55:39 -07002925 global_page_state(NR_FILE_DIRTY),
Christoph Lameterce866b32006-06-30 01:55:40 -07002926 global_page_state(NR_WRITEBACK),
Christoph Lameterfd39fc82006-06-30 01:55:40 -07002927 global_page_state(NR_UNSTABLE_NFS),
Christoph Lameterd23ad422007-02-10 01:43:02 -08002928 global_page_state(NR_FREE_PAGES),
KOSAKI Motohiro3701b032009-09-21 17:01:29 -07002929 global_page_state(NR_SLAB_RECLAIMABLE),
2930 global_page_state(NR_SLAB_UNRECLAIMABLE),
Christoph Lameter65ba55f2006-06-30 01:55:34 -07002931 global_page_state(NR_FILE_MAPPED),
KOSAKI Motohiro4b021082009-09-21 17:01:33 -07002932 global_page_state(NR_SHMEM),
Andrew Mortona25700a2007-02-08 14:20:40 -08002933 global_page_state(NR_PAGETABLE),
Larry Bassel6d6e2c92012-12-14 14:21:05 -08002934 global_page_state(NR_BOUNCE),
2935 global_page_state(NR_FREE_CMA_PAGES));
Linus Torvalds1da177e2005-04-16 15:20:36 -07002936
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07002937 for_each_populated_zone(zone) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07002938 int i;
2939
David Rientjes7bf02ea2011-05-24 17:11:16 -07002940 if (skip_free_areas_node(filter, zone_to_nid(zone)))
David Rientjesddd588b2011-03-22 16:30:46 -07002941 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002942 show_node(zone);
2943 printk("%s"
2944 " free:%lukB"
2945 " min:%lukB"
2946 " low:%lukB"
2947 " high:%lukB"
Rik van Riel4f98a2f2008-10-18 20:26:32 -07002948 " active_anon:%lukB"
2949 " inactive_anon:%lukB"
2950 " active_file:%lukB"
2951 " inactive_file:%lukB"
Lee Schermerhorn7b854122008-10-18 20:26:40 -07002952 " unevictable:%lukB"
KOSAKI Motohiroa7312862009-09-21 17:01:37 -07002953 " isolated(anon):%lukB"
2954 " isolated(file):%lukB"
Linus Torvalds1da177e2005-04-16 15:20:36 -07002955 " present:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07002956 " mlocked:%lukB"
2957 " dirty:%lukB"
2958 " writeback:%lukB"
2959 " mapped:%lukB"
KOSAKI Motohiro4b021082009-09-21 17:01:33 -07002960 " shmem:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07002961 " slab_reclaimable:%lukB"
2962 " slab_unreclaimable:%lukB"
KOSAKI Motohiroc6a7f572009-09-21 17:01:32 -07002963 " kernel_stack:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07002964 " pagetables:%lukB"
2965 " unstable:%lukB"
2966 " bounce:%lukB"
Larry Bassel6d6e2c92012-12-14 14:21:05 -08002967 " free_cma:%lukB"
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07002968 " writeback_tmp:%lukB"
Linus Torvalds1da177e2005-04-16 15:20:36 -07002969 " pages_scanned:%lu"
2970 " all_unreclaimable? %s"
2971 "\n",
2972 zone->name,
Mel Gorman88f5acf2011-01-13 15:45:41 -08002973 K(zone_page_state(zone, NR_FREE_PAGES)),
Mel Gorman41858962009-06-16 15:32:12 -07002974 K(min_wmark_pages(zone)),
2975 K(low_wmark_pages(zone)),
2976 K(high_wmark_pages(zone)),
Rik van Riel4f98a2f2008-10-18 20:26:32 -07002977 K(zone_page_state(zone, NR_ACTIVE_ANON)),
2978 K(zone_page_state(zone, NR_INACTIVE_ANON)),
2979 K(zone_page_state(zone, NR_ACTIVE_FILE)),
2980 K(zone_page_state(zone, NR_INACTIVE_FILE)),
Lee Schermerhorn7b854122008-10-18 20:26:40 -07002981 K(zone_page_state(zone, NR_UNEVICTABLE)),
KOSAKI Motohiroa7312862009-09-21 17:01:37 -07002982 K(zone_page_state(zone, NR_ISOLATED_ANON)),
2983 K(zone_page_state(zone, NR_ISOLATED_FILE)),
Linus Torvalds1da177e2005-04-16 15:20:36 -07002984 K(zone->present_pages),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07002985 K(zone_page_state(zone, NR_MLOCK)),
2986 K(zone_page_state(zone, NR_FILE_DIRTY)),
2987 K(zone_page_state(zone, NR_WRITEBACK)),
2988 K(zone_page_state(zone, NR_FILE_MAPPED)),
KOSAKI Motohiro4b021082009-09-21 17:01:33 -07002989 K(zone_page_state(zone, NR_SHMEM)),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07002990 K(zone_page_state(zone, NR_SLAB_RECLAIMABLE)),
2991 K(zone_page_state(zone, NR_SLAB_UNRECLAIMABLE)),
KOSAKI Motohiroc6a7f572009-09-21 17:01:32 -07002992 zone_page_state(zone, NR_KERNEL_STACK) *
2993 THREAD_SIZE / 1024,
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07002994 K(zone_page_state(zone, NR_PAGETABLE)),
2995 K(zone_page_state(zone, NR_UNSTABLE_NFS)),
2996 K(zone_page_state(zone, NR_BOUNCE)),
Larry Bassel6d6e2c92012-12-14 14:21:05 -08002997 K(zone_page_state(zone, NR_FREE_CMA_PAGES)),
KOSAKI Motohiro4a0aa732009-09-21 17:01:30 -07002998 K(zone_page_state(zone, NR_WRITEBACK_TEMP)),
Linus Torvalds1da177e2005-04-16 15:20:36 -07002999 zone->pages_scanned,
KOSAKI Motohiro93e4a892010-03-05 13:41:55 -08003000 (zone->all_unreclaimable ? "yes" : "no")
Linus Torvalds1da177e2005-04-16 15:20:36 -07003001 );
3002 printk("lowmem_reserve[]:");
3003 for (i = 0; i < MAX_NR_ZONES; i++)
3004 printk(" %lu", zone->lowmem_reserve[i]);
3005 printk("\n");
3006 }
3007
KOSAKI Motohiroee99c712009-03-31 15:19:31 -07003008 for_each_populated_zone(zone) {
Kirill Korotaev8f9de512006-06-23 02:03:50 -07003009 unsigned long nr[MAX_ORDER], flags, order, total = 0;
Rabin Vincent394b8082012-12-11 16:00:24 -08003010 unsigned char types[MAX_ORDER];
Linus Torvalds1da177e2005-04-16 15:20:36 -07003011
David Rientjes7bf02ea2011-05-24 17:11:16 -07003012 if (skip_free_areas_node(filter, zone_to_nid(zone)))
David Rientjesddd588b2011-03-22 16:30:46 -07003013 continue;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003014 show_node(zone);
3015 printk("%s: ", zone->name);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003016
3017 spin_lock_irqsave(&zone->lock, flags);
3018 for (order = 0; order < MAX_ORDER; order++) {
Rabin Vincent394b8082012-12-11 16:00:24 -08003019 struct free_area *area = &zone->free_area[order];
3020 int type;
3021
3022 nr[order] = area->nr_free;
Kirill Korotaev8f9de512006-06-23 02:03:50 -07003023 total += nr[order] << order;
Rabin Vincent394b8082012-12-11 16:00:24 -08003024
3025 types[order] = 0;
3026 for (type = 0; type < MIGRATE_TYPES; type++) {
3027 if (!list_empty(&area->free_list[type]))
3028 types[order] |= 1 << type;
3029 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003030 }
3031 spin_unlock_irqrestore(&zone->lock, flags);
Rabin Vincent394b8082012-12-11 16:00:24 -08003032 for (order = 0; order < MAX_ORDER; order++) {
Kirill Korotaev8f9de512006-06-23 02:03:50 -07003033 printk("%lu*%lukB ", nr[order], K(1UL) << order);
Rabin Vincent394b8082012-12-11 16:00:24 -08003034 if (nr[order])
3035 show_migration_types(types[order]);
3036 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003037 printk("= %lukB\n", K(total));
3038 }
3039
Larry Woodmane6f36022008-02-04 22:29:30 -08003040 printk("%ld total pagecache pages\n", global_page_state(NR_FILE_PAGES));
3041
Linus Torvalds1da177e2005-04-16 15:20:36 -07003042 show_swap_cache_info();
3043}
3044
Mel Gorman19770b32008-04-28 02:12:18 -07003045static void zoneref_set_zone(struct zone *zone, struct zoneref *zoneref)
3046{
3047 zoneref->zone = zone;
3048 zoneref->zone_idx = zone_idx(zone);
3049}
3050
Linus Torvalds1da177e2005-04-16 15:20:36 -07003051/*
3052 * Builds allocation fallback zone lists.
Christoph Lameter1a932052006-01-06 00:11:16 -08003053 *
3054 * Add all populated zones of a node to the zonelist.
Linus Torvalds1da177e2005-04-16 15:20:36 -07003055 */
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003056static int build_zonelists_node(pg_data_t *pgdat, struct zonelist *zonelist,
3057 int nr_zones, enum zone_type zone_type)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003058{
Christoph Lameter1a932052006-01-06 00:11:16 -08003059 struct zone *zone;
3060
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07003061 BUG_ON(zone_type >= MAX_NR_ZONES);
Christoph Lameter2f6726e2006-09-25 23:31:18 -07003062 zone_type++;
Christoph Lameter02a68a52006-01-06 00:11:18 -08003063
3064 do {
Christoph Lameter2f6726e2006-09-25 23:31:18 -07003065 zone_type--;
Christoph Lameter070f8032006-01-06 00:11:19 -08003066 zone = pgdat->node_zones + zone_type;
Christoph Lameter1a932052006-01-06 00:11:16 -08003067 if (populated_zone(zone)) {
Mel Gormandd1a2392008-04-28 02:12:17 -07003068 zoneref_set_zone(zone,
3069 &zonelist->_zonerefs[nr_zones++]);
Christoph Lameter070f8032006-01-06 00:11:19 -08003070 check_highest_zone(zone_type);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003071 }
Christoph Lameter02a68a52006-01-06 00:11:18 -08003072
Christoph Lameter2f6726e2006-09-25 23:31:18 -07003073 } while (zone_type);
Christoph Lameter070f8032006-01-06 00:11:19 -08003074 return nr_zones;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003075}
3076
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003077
3078/*
3079 * zonelist_order:
3080 * 0 = automatic detection of better ordering.
3081 * 1 = order by ([node] distance, -zonetype)
3082 * 2 = order by (-zonetype, [node] distance)
3083 *
3084 * If not NUMA, ZONELIST_ORDER_ZONE and ZONELIST_ORDER_NODE will create
3085 * the same zonelist. So only NUMA can configure this param.
3086 */
3087#define ZONELIST_ORDER_DEFAULT 0
3088#define ZONELIST_ORDER_NODE 1
3089#define ZONELIST_ORDER_ZONE 2
3090
3091/* zonelist order in the kernel.
3092 * set_zonelist_order() will set this to NODE or ZONE.
3093 */
3094static int current_zonelist_order = ZONELIST_ORDER_DEFAULT;
3095static char zonelist_order_name[3][8] = {"Default", "Node", "Zone"};
3096
3097
Linus Torvalds1da177e2005-04-16 15:20:36 -07003098#ifdef CONFIG_NUMA
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003099/* The value user specified ....changed by config */
3100static int user_zonelist_order = ZONELIST_ORDER_DEFAULT;
3101/* string for sysctl */
3102#define NUMA_ZONELIST_ORDER_LEN 16
3103char numa_zonelist_order[16] = "default";
3104
3105/*
3106 * interface for configure zonelist ordering.
3107 * command line option "numa_zonelist_order"
3108 * = "[dD]efault - default, automatic configuration.
3109 * = "[nN]ode - order by node locality, then by zone within node
3110 * = "[zZ]one - order by zone, then by locality within zone
3111 */
3112
3113static int __parse_numa_zonelist_order(char *s)
3114{
3115 if (*s == 'd' || *s == 'D') {
3116 user_zonelist_order = ZONELIST_ORDER_DEFAULT;
3117 } else if (*s == 'n' || *s == 'N') {
3118 user_zonelist_order = ZONELIST_ORDER_NODE;
3119 } else if (*s == 'z' || *s == 'Z') {
3120 user_zonelist_order = ZONELIST_ORDER_ZONE;
3121 } else {
3122 printk(KERN_WARNING
3123 "Ignoring invalid numa_zonelist_order value: "
3124 "%s\n", s);
3125 return -EINVAL;
3126 }
3127 return 0;
3128}
3129
3130static __init int setup_numa_zonelist_order(char *s)
3131{
Volodymyr G. Lukiianykecb256f2011-01-13 15:46:26 -08003132 int ret;
3133
3134 if (!s)
3135 return 0;
3136
3137 ret = __parse_numa_zonelist_order(s);
3138 if (ret == 0)
3139 strlcpy(numa_zonelist_order, s, NUMA_ZONELIST_ORDER_LEN);
3140
3141 return ret;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003142}
3143early_param("numa_zonelist_order", setup_numa_zonelist_order);
3144
3145/*
3146 * sysctl handler for numa_zonelist_order
3147 */
3148int numa_zonelist_order_handler(ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07003149 void __user *buffer, size_t *length,
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003150 loff_t *ppos)
3151{
3152 char saved_string[NUMA_ZONELIST_ORDER_LEN];
3153 int ret;
Andi Kleen443c6f12009-12-23 21:00:47 +01003154 static DEFINE_MUTEX(zl_order_mutex);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003155
Andi Kleen443c6f12009-12-23 21:00:47 +01003156 mutex_lock(&zl_order_mutex);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003157 if (write)
Andi Kleen443c6f12009-12-23 21:00:47 +01003158 strcpy(saved_string, (char*)table->data);
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07003159 ret = proc_dostring(table, write, buffer, length, ppos);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003160 if (ret)
Andi Kleen443c6f12009-12-23 21:00:47 +01003161 goto out;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003162 if (write) {
3163 int oldval = user_zonelist_order;
3164 if (__parse_numa_zonelist_order((char*)table->data)) {
3165 /*
3166 * bogus value. restore saved string
3167 */
3168 strncpy((char*)table->data, saved_string,
3169 NUMA_ZONELIST_ORDER_LEN);
3170 user_zonelist_order = oldval;
Haicheng Li4eaf3f62010-05-24 14:32:52 -07003171 } else if (oldval != user_zonelist_order) {
3172 mutex_lock(&zonelists_mutex);
Haicheng Li1f522502010-05-24 14:32:51 -07003173 build_all_zonelists(NULL);
Haicheng Li4eaf3f62010-05-24 14:32:52 -07003174 mutex_unlock(&zonelists_mutex);
3175 }
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003176 }
Andi Kleen443c6f12009-12-23 21:00:47 +01003177out:
3178 mutex_unlock(&zl_order_mutex);
3179 return ret;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003180}
3181
3182
Christoph Lameter62bc62a2009-06-16 15:32:15 -07003183#define MAX_NODE_LOAD (nr_online_nodes)
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003184static int node_load[MAX_NUMNODES];
3185
Linus Torvalds1da177e2005-04-16 15:20:36 -07003186/**
Pavel Pisa4dc3b162005-05-01 08:59:25 -07003187 * find_next_best_node - find the next node that should appear in a given node's fallback list
Linus Torvalds1da177e2005-04-16 15:20:36 -07003188 * @node: node whose fallback list we're appending
3189 * @used_node_mask: nodemask_t of already used nodes
3190 *
3191 * We use a number of factors to determine which is the next node that should
3192 * appear on a given node's fallback list. The node should not have appeared
3193 * already in @node's fallback list, and it should be the next closest node
3194 * according to the distance array (which contains arbitrary distance values
3195 * from each node to each node in the system), and should also prefer nodes
3196 * with no CPUs, since presumably they'll have very little allocation pressure
3197 * on them otherwise.
3198 * It returns -1 if no node is found.
3199 */
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003200static int find_next_best_node(int node, nodemask_t *used_node_mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003201{
Linus Torvalds4cf808e2006-02-17 20:38:21 +01003202 int n, val;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003203 int min_val = INT_MAX;
3204 int best_node = -1;
Rusty Russella70f7302009-03-13 14:49:46 +10303205 const struct cpumask *tmp = cpumask_of_node(0);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003206
Linus Torvalds4cf808e2006-02-17 20:38:21 +01003207 /* Use the local node if we haven't already */
3208 if (!node_isset(node, *used_node_mask)) {
3209 node_set(node, *used_node_mask);
3210 return node;
3211 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003212
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07003213 for_each_node_state(n, N_HIGH_MEMORY) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003214
3215 /* Don't want a node to appear more than once */
3216 if (node_isset(n, *used_node_mask))
3217 continue;
3218
Linus Torvalds1da177e2005-04-16 15:20:36 -07003219 /* Use the distance array to find the distance */
3220 val = node_distance(node, n);
3221
Linus Torvalds4cf808e2006-02-17 20:38:21 +01003222 /* Penalize nodes under us ("prefer the next node") */
3223 val += (n < node);
3224
Linus Torvalds1da177e2005-04-16 15:20:36 -07003225 /* Give preference to headless and unused nodes */
Rusty Russella70f7302009-03-13 14:49:46 +10303226 tmp = cpumask_of_node(n);
3227 if (!cpumask_empty(tmp))
Linus Torvalds1da177e2005-04-16 15:20:36 -07003228 val += PENALTY_FOR_NODE_WITH_CPUS;
3229
3230 /* Slight preference for less loaded node */
3231 val *= (MAX_NODE_LOAD*MAX_NUMNODES);
3232 val += node_load[n];
3233
3234 if (val < min_val) {
3235 min_val = val;
3236 best_node = n;
3237 }
3238 }
3239
3240 if (best_node >= 0)
3241 node_set(best_node, *used_node_mask);
3242
3243 return best_node;
3244}
3245
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003246
3247/*
3248 * Build zonelists ordered by node and zones within node.
3249 * This results in maximum locality--normal zone overflows into local
3250 * DMA zone, if any--but risks exhausting DMA zone.
3251 */
3252static void build_zonelists_in_node_order(pg_data_t *pgdat, int node)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003253{
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003254 int j;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003255 struct zonelist *zonelist;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003256
Mel Gorman54a6eb52008-04-28 02:12:16 -07003257 zonelist = &pgdat->node_zonelists[0];
Mel Gormandd1a2392008-04-28 02:12:17 -07003258 for (j = 0; zonelist->_zonerefs[j].zone != NULL; j++)
Mel Gorman54a6eb52008-04-28 02:12:16 -07003259 ;
3260 j = build_zonelists_node(NODE_DATA(node), zonelist, j,
3261 MAX_NR_ZONES - 1);
Mel Gormandd1a2392008-04-28 02:12:17 -07003262 zonelist->_zonerefs[j].zone = NULL;
3263 zonelist->_zonerefs[j].zone_idx = 0;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003264}
3265
3266/*
Christoph Lameter523b9452007-10-16 01:25:37 -07003267 * Build gfp_thisnode zonelists
3268 */
3269static void build_thisnode_zonelists(pg_data_t *pgdat)
3270{
Christoph Lameter523b9452007-10-16 01:25:37 -07003271 int j;
3272 struct zonelist *zonelist;
3273
Mel Gorman54a6eb52008-04-28 02:12:16 -07003274 zonelist = &pgdat->node_zonelists[1];
3275 j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
Mel Gormandd1a2392008-04-28 02:12:17 -07003276 zonelist->_zonerefs[j].zone = NULL;
3277 zonelist->_zonerefs[j].zone_idx = 0;
Christoph Lameter523b9452007-10-16 01:25:37 -07003278}
3279
3280/*
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003281 * Build zonelists ordered by zone and nodes within zones.
3282 * This results in conserving DMA zone[s] until all Normal memory is
3283 * exhausted, but results in overflowing to remote node while memory
3284 * may still exist in local DMA zone.
3285 */
3286static int node_order[MAX_NUMNODES];
3287
3288static void build_zonelists_in_zone_order(pg_data_t *pgdat, int nr_nodes)
3289{
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003290 int pos, j, node;
3291 int zone_type; /* needs to be signed */
3292 struct zone *z;
3293 struct zonelist *zonelist;
3294
Mel Gorman54a6eb52008-04-28 02:12:16 -07003295 zonelist = &pgdat->node_zonelists[0];
3296 pos = 0;
3297 for (zone_type = MAX_NR_ZONES - 1; zone_type >= 0; zone_type--) {
3298 for (j = 0; j < nr_nodes; j++) {
3299 node = node_order[j];
3300 z = &NODE_DATA(node)->node_zones[zone_type];
3301 if (populated_zone(z)) {
Mel Gormandd1a2392008-04-28 02:12:17 -07003302 zoneref_set_zone(z,
3303 &zonelist->_zonerefs[pos++]);
Mel Gorman54a6eb52008-04-28 02:12:16 -07003304 check_highest_zone(zone_type);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003305 }
3306 }
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003307 }
Mel Gormandd1a2392008-04-28 02:12:17 -07003308 zonelist->_zonerefs[pos].zone = NULL;
3309 zonelist->_zonerefs[pos].zone_idx = 0;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003310}
3311
3312static int default_zonelist_order(void)
3313{
3314 int nid, zone_type;
3315 unsigned long low_kmem_size,total_size;
3316 struct zone *z;
3317 int average_size;
3318 /*
Thomas Weber88393162010-03-16 11:47:56 +01003319 * ZONE_DMA and ZONE_DMA32 can be very small area in the system.
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003320 * If they are really small and used heavily, the system can fall
3321 * into OOM very easily.
David Rientjese325c902010-05-24 14:32:13 -07003322 * This function detect ZONE_DMA/DMA32 size and configures zone order.
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003323 */
3324 /* Is there ZONE_NORMAL ? (ex. ppc has only DMA zone..) */
3325 low_kmem_size = 0;
3326 total_size = 0;
3327 for_each_online_node(nid) {
3328 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
3329 z = &NODE_DATA(nid)->node_zones[zone_type];
3330 if (populated_zone(z)) {
3331 if (zone_type < ZONE_NORMAL)
3332 low_kmem_size += z->present_pages;
3333 total_size += z->present_pages;
David Rientjese325c902010-05-24 14:32:13 -07003334 } else if (zone_type == ZONE_NORMAL) {
3335 /*
3336 * If any node has only lowmem, then node order
3337 * is preferred to allow kernel allocations
3338 * locally; otherwise, they can easily infringe
3339 * on other nodes when there is an abundance of
3340 * lowmem available to allocate from.
3341 */
3342 return ZONELIST_ORDER_NODE;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003343 }
3344 }
3345 }
3346 if (!low_kmem_size || /* there are no DMA area. */
3347 low_kmem_size > total_size/2) /* DMA/DMA32 is big. */
3348 return ZONELIST_ORDER_NODE;
3349 /*
3350 * look into each node's config.
3351 * If there is a node whose DMA/DMA32 memory is very big area on
3352 * local memory, NODE_ORDER may be suitable.
3353 */
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07003354 average_size = total_size /
3355 (nodes_weight(node_states[N_HIGH_MEMORY]) + 1);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003356 for_each_online_node(nid) {
3357 low_kmem_size = 0;
3358 total_size = 0;
3359 for (zone_type = 0; zone_type < MAX_NR_ZONES; zone_type++) {
3360 z = &NODE_DATA(nid)->node_zones[zone_type];
3361 if (populated_zone(z)) {
3362 if (zone_type < ZONE_NORMAL)
3363 low_kmem_size += z->present_pages;
3364 total_size += z->present_pages;
3365 }
3366 }
3367 if (low_kmem_size &&
3368 total_size > average_size && /* ignore small node */
3369 low_kmem_size > total_size * 70/100)
3370 return ZONELIST_ORDER_NODE;
3371 }
3372 return ZONELIST_ORDER_ZONE;
3373}
3374
3375static void set_zonelist_order(void)
3376{
3377 if (user_zonelist_order == ZONELIST_ORDER_DEFAULT)
3378 current_zonelist_order = default_zonelist_order();
3379 else
3380 current_zonelist_order = user_zonelist_order;
3381}
3382
3383static void build_zonelists(pg_data_t *pgdat)
3384{
3385 int j, node, load;
3386 enum zone_type i;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003387 nodemask_t used_mask;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003388 int local_node, prev_node;
3389 struct zonelist *zonelist;
3390 int order = current_zonelist_order;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003391
3392 /* initialize zonelists */
Christoph Lameter523b9452007-10-16 01:25:37 -07003393 for (i = 0; i < MAX_ZONELISTS; i++) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07003394 zonelist = pgdat->node_zonelists + i;
Mel Gormandd1a2392008-04-28 02:12:17 -07003395 zonelist->_zonerefs[0].zone = NULL;
3396 zonelist->_zonerefs[0].zone_idx = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003397 }
3398
3399 /* NUMA-aware ordering of nodes */
3400 local_node = pgdat->node_id;
Christoph Lameter62bc62a2009-06-16 15:32:15 -07003401 load = nr_online_nodes;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003402 prev_node = local_node;
3403 nodes_clear(used_mask);
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003404
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003405 memset(node_order, 0, sizeof(node_order));
3406 j = 0;
3407
Linus Torvalds1da177e2005-04-16 15:20:36 -07003408 while ((node = find_next_best_node(local_node, &used_mask)) >= 0) {
Christoph Lameter9eeff232006-01-18 17:42:31 -08003409 int distance = node_distance(local_node, node);
3410
3411 /*
3412 * If another node is sufficiently far away then it is better
3413 * to reclaim pages in a zone before going off node.
3414 */
3415 if (distance > RECLAIM_DISTANCE)
3416 zone_reclaim_mode = 1;
3417
Linus Torvalds1da177e2005-04-16 15:20:36 -07003418 /*
3419 * We don't want to pressure a particular node.
3420 * So adding penalty to the first node in same
3421 * distance group to make it round-robin.
3422 */
Christoph Lameter9eeff232006-01-18 17:42:31 -08003423 if (distance != node_distance(local_node, prev_node))
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003424 node_load[node] = load;
3425
Linus Torvalds1da177e2005-04-16 15:20:36 -07003426 prev_node = node;
3427 load--;
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003428 if (order == ZONELIST_ORDER_NODE)
3429 build_zonelists_in_node_order(pgdat, node);
3430 else
3431 node_order[j++] = node; /* remember order */
3432 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07003433
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003434 if (order == ZONELIST_ORDER_ZONE) {
3435 /* calculate node order -- i.e., DMA last! */
3436 build_zonelists_in_zone_order(pgdat, j);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003437 }
Christoph Lameter523b9452007-10-16 01:25:37 -07003438
3439 build_thisnode_zonelists(pgdat);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003440}
3441
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003442/* Construct the zonelist performance cache - see further mmzone.h */
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003443static void build_zonelist_cache(pg_data_t *pgdat)
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003444{
Mel Gorman54a6eb52008-04-28 02:12:16 -07003445 struct zonelist *zonelist;
3446 struct zonelist_cache *zlc;
Mel Gormandd1a2392008-04-28 02:12:17 -07003447 struct zoneref *z;
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003448
Mel Gorman54a6eb52008-04-28 02:12:16 -07003449 zonelist = &pgdat->node_zonelists[0];
3450 zonelist->zlcache_ptr = zlc = &zonelist->zlcache;
3451 bitmap_zero(zlc->fullzones, MAX_ZONES_PER_ZONELIST);
Mel Gormandd1a2392008-04-28 02:12:17 -07003452 for (z = zonelist->_zonerefs; z->zone; z++)
3453 zlc->z_to_n[z - zonelist->_zonerefs] = zonelist_node_idx(z);
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003454}
3455
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07003456#ifdef CONFIG_HAVE_MEMORYLESS_NODES
3457/*
3458 * Return node id of node used for "local" allocations.
3459 * I.e., first node id of first zone in arg node's generic zonelist.
3460 * Used for initializing percpu 'numa_mem', which is used primarily
3461 * for kernel allocations, so use GFP_KERNEL flags to locate zonelist.
3462 */
3463int local_memory_node(int node)
3464{
3465 struct zone *zone;
3466
3467 (void)first_zones_zonelist(node_zonelist(node, GFP_KERNEL),
3468 gfp_zone(GFP_KERNEL),
3469 NULL,
3470 &zone);
3471 return zone->node;
3472}
3473#endif
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003474
Linus Torvalds1da177e2005-04-16 15:20:36 -07003475#else /* CONFIG_NUMA */
3476
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003477static void set_zonelist_order(void)
3478{
3479 current_zonelist_order = ZONELIST_ORDER_ZONE;
3480}
3481
3482static void build_zonelists(pg_data_t *pgdat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003483{
Christoph Lameter19655d32006-09-25 23:31:19 -07003484 int node, local_node;
Mel Gorman54a6eb52008-04-28 02:12:16 -07003485 enum zone_type j;
3486 struct zonelist *zonelist;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003487
3488 local_node = pgdat->node_id;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003489
Mel Gorman54a6eb52008-04-28 02:12:16 -07003490 zonelist = &pgdat->node_zonelists[0];
3491 j = build_zonelists_node(pgdat, zonelist, 0, MAX_NR_ZONES - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003492
Mel Gorman54a6eb52008-04-28 02:12:16 -07003493 /*
3494 * Now we build the zonelist so that it contains the zones
3495 * of all the other nodes.
3496 * We don't want to pressure a particular node, so when
3497 * building the zones for node N, we make sure that the
3498 * zones coming right after the local ones are those from
3499 * node N+1 (modulo N)
3500 */
3501 for (node = local_node + 1; node < MAX_NUMNODES; node++) {
3502 if (!node_online(node))
3503 continue;
3504 j = build_zonelists_node(NODE_DATA(node), zonelist, j,
3505 MAX_NR_ZONES - 1);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003506 }
Mel Gorman54a6eb52008-04-28 02:12:16 -07003507 for (node = 0; node < local_node; node++) {
3508 if (!node_online(node))
3509 continue;
3510 j = build_zonelists_node(NODE_DATA(node), zonelist, j,
3511 MAX_NR_ZONES - 1);
3512 }
3513
Mel Gormandd1a2392008-04-28 02:12:17 -07003514 zonelist->_zonerefs[j].zone = NULL;
3515 zonelist->_zonerefs[j].zone_idx = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003516}
3517
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003518/* non-NUMA variant of zonelist performance cache - just NULL zlcache_ptr */
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003519static void build_zonelist_cache(pg_data_t *pgdat)
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003520{
Mel Gorman54a6eb52008-04-28 02:12:16 -07003521 pgdat->node_zonelists[0].zlcache_ptr = NULL;
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003522}
3523
Linus Torvalds1da177e2005-04-16 15:20:36 -07003524#endif /* CONFIG_NUMA */
3525
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09003526/*
3527 * Boot pageset table. One per cpu which is going to be used for all
3528 * zones and all nodes. The parameters will be set in such a way
3529 * that an item put on a list will immediately be handed over to
3530 * the buddy list. This is safe since pageset manipulation is done
3531 * with interrupts disabled.
3532 *
3533 * The boot_pagesets must be kept even after bootup is complete for
3534 * unused processors and/or zones. They do play a role for bootstrapping
3535 * hotplugged processors.
3536 *
3537 * zoneinfo_show() and maybe other functions do
3538 * not check if the processor is online before following the pageset pointer.
3539 * Other parts of the kernel may not check if the zone is available.
3540 */
3541static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch);
3542static DEFINE_PER_CPU(struct per_cpu_pageset, boot_pageset);
Haicheng Li1f522502010-05-24 14:32:51 -07003543static void setup_zone_pageset(struct zone *zone);
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09003544
Haicheng Li4eaf3f62010-05-24 14:32:52 -07003545/*
3546 * Global mutex to protect against size modification of zonelists
3547 * as well as to serialize pageset setup for the new populated zone.
3548 */
3549DEFINE_MUTEX(zonelists_mutex);
3550
Rusty Russell9b1a4d32008-07-28 12:16:30 -05003551/* return values int ....just for stop_machine() */
Haicheng Li1f522502010-05-24 14:32:51 -07003552static __init_refok int __build_all_zonelists(void *data)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003553{
Yasunori Goto68113782006-06-23 02:03:11 -07003554 int nid;
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09003555 int cpu;
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003556
Bo Liu7f9cfb32009-08-18 14:11:19 -07003557#ifdef CONFIG_NUMA
3558 memset(node_load, 0, sizeof(node_load));
3559#endif
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003560 for_each_online_node(nid) {
Christoph Lameter7ea15302007-10-16 01:25:29 -07003561 pg_data_t *pgdat = NODE_DATA(nid);
3562
3563 build_zonelists(pgdat);
3564 build_zonelist_cache(pgdat);
Paul Jackson9276b1bc2006-12-06 20:31:48 -08003565 }
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09003566
3567 /*
3568 * Initialize the boot_pagesets that are going to be used
3569 * for bootstrapping processors. The real pagesets for
3570 * each zone will be allocated later when the per cpu
3571 * allocator is available.
3572 *
3573 * boot_pagesets are used also for bootstrapping offline
3574 * cpus if the system is already booted because the pagesets
3575 * are needed to initialize allocators on a specific cpu too.
3576 * F.e. the percpu allocator needs the page allocator which
3577 * needs the percpu allocator in order to allocate its pagesets
3578 * (a chicken-egg dilemma).
3579 */
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07003580 for_each_possible_cpu(cpu) {
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09003581 setup_pageset(&per_cpu(boot_pageset, cpu), 0);
3582
Lee Schermerhorn7aac7892010-05-26 14:45:00 -07003583#ifdef CONFIG_HAVE_MEMORYLESS_NODES
3584 /*
3585 * We now know the "local memory node" for each node--
3586 * i.e., the node of the first zone in the generic zonelist.
3587 * Set up numa_mem percpu variable for on-line cpus. During
3588 * boot, only the boot cpu should be on-line; we'll init the
3589 * secondary cpus' numa_mem as they come on-line. During
3590 * node/memory hotplug, we'll fixup all on-line cpus.
3591 */
3592 if (cpu_online(cpu))
3593 set_cpu_numa_mem(cpu, local_memory_node(cpu_to_node(cpu)));
3594#endif
3595 }
3596
Yasunori Goto68113782006-06-23 02:03:11 -07003597 return 0;
3598}
Linus Torvalds1da177e2005-04-16 15:20:36 -07003599
Haicheng Li4eaf3f62010-05-24 14:32:52 -07003600/*
3601 * Called with zonelists_mutex held always
3602 * unless system_state == SYSTEM_BOOTING.
3603 */
Paul Mundt9f6ae442011-04-14 15:21:57 -07003604void __ref build_all_zonelists(void *data)
Yasunori Goto68113782006-06-23 02:03:11 -07003605{
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003606 set_zonelist_order();
3607
Yasunori Goto68113782006-06-23 02:03:11 -07003608 if (system_state == SYSTEM_BOOTING) {
Randy Dunlap423b41d2006-09-27 01:50:12 -07003609 __build_all_zonelists(NULL);
Mel Gorman68ad8df2008-07-23 21:26:52 -07003610 mminit_verify_zonelist();
Yasunori Goto68113782006-06-23 02:03:11 -07003611 cpuset_init_current_mems_allowed();
3612 } else {
Simon Arlott183ff222007-10-20 01:27:18 +02003613 /* we have to stop all cpus to guarantee there is no user
Yasunori Goto68113782006-06-23 02:03:11 -07003614 of zonelist */
KAMEZAWA Hiroyukie9959f02010-11-24 12:57:09 -08003615#ifdef CONFIG_MEMORY_HOTPLUG
3616 if (data)
3617 setup_zone_pageset((struct zone *)data);
3618#endif
3619 stop_machine(__build_all_zonelists, NULL, NULL);
Yasunori Goto68113782006-06-23 02:03:11 -07003620 /* cpuset refresh routine should be here */
3621 }
Andrew Mortonbd1e22b2006-06-23 02:03:47 -07003622 vm_total_pages = nr_free_pagecache_pages();
Mel Gorman9ef9acb2007-10-16 01:25:54 -07003623 /*
3624 * Disable grouping by mobility if the number of pages in the
3625 * system is too low to allow the mechanism to work. It would be
3626 * more accurate, but expensive to check per-zone. This check is
3627 * made on memory-hotadd so a system can start with mobility
3628 * disabled and enable it later
3629 */
Mel Gormand9c23402007-10-16 01:26:01 -07003630 if (vm_total_pages < (pageblock_nr_pages * MIGRATE_TYPES))
Mel Gorman9ef9acb2007-10-16 01:25:54 -07003631 page_group_by_mobility_disabled = 1;
3632 else
3633 page_group_by_mobility_disabled = 0;
3634
3635 printk("Built %i zonelists in %s order, mobility grouping %s. "
3636 "Total pages: %ld\n",
Christoph Lameter62bc62a2009-06-16 15:32:15 -07003637 nr_online_nodes,
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003638 zonelist_order_name[current_zonelist_order],
Mel Gorman9ef9acb2007-10-16 01:25:54 -07003639 page_group_by_mobility_disabled ? "off" : "on",
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -07003640 vm_total_pages);
3641#ifdef CONFIG_NUMA
3642 printk("Policy zone: %s\n", zone_names[policy_zone]);
3643#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003644}
3645
3646/*
3647 * Helper functions to size the waitqueue hash table.
3648 * Essentially these want to choose hash table sizes sufficiently
3649 * large so that collisions trying to wait on pages are rare.
3650 * But in fact, the number of active page waitqueues on typical
3651 * systems is ridiculously low, less than 200. So this is even
3652 * conservative, even though it seems large.
3653 *
3654 * The constant PAGES_PER_WAITQUEUE specifies the ratio of pages to
3655 * waitqueues, i.e. the size of the waitq table given the number of pages.
3656 */
3657#define PAGES_PER_WAITQUEUE 256
3658
Yasunori Gotocca448f2006-06-23 02:03:10 -07003659#ifndef CONFIG_MEMORY_HOTPLUG
Yasunori Goto02b694d2006-06-23 02:03:08 -07003660static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003661{
3662 unsigned long size = 1;
3663
3664 pages /= PAGES_PER_WAITQUEUE;
3665
3666 while (size < pages)
3667 size <<= 1;
3668
3669 /*
3670 * Once we have dozens or even hundreds of threads sleeping
3671 * on IO we've got bigger problems than wait queue collision.
3672 * Limit the size of the wait table to a reasonable size.
3673 */
3674 size = min(size, 4096UL);
3675
3676 return max(size, 4UL);
3677}
Yasunori Gotocca448f2006-06-23 02:03:10 -07003678#else
3679/*
3680 * A zone's size might be changed by hot-add, so it is not possible to determine
3681 * a suitable size for its wait_table. So we use the maximum size now.
3682 *
3683 * The max wait table size = 4096 x sizeof(wait_queue_head_t). ie:
3684 *
3685 * i386 (preemption config) : 4096 x 16 = 64Kbyte.
3686 * ia64, x86-64 (no preemption): 4096 x 20 = 80Kbyte.
3687 * ia64, x86-64 (preemption) : 4096 x 24 = 96Kbyte.
3688 *
3689 * The maximum entries are prepared when a zone's memory is (512K + 256) pages
3690 * or more by the traditional way. (See above). It equals:
3691 *
3692 * i386, x86-64, powerpc(4K page size) : = ( 2G + 1M)byte.
3693 * ia64(16K page size) : = ( 8G + 4M)byte.
3694 * powerpc (64K page size) : = (32G +16M)byte.
3695 */
3696static inline unsigned long wait_table_hash_nr_entries(unsigned long pages)
3697{
3698 return 4096UL;
3699}
3700#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003701
3702/*
3703 * This is an integer logarithm so that shifts can be used later
3704 * to extract the more random high bits from the multiplicative
3705 * hash function before the remainder is taken.
3706 */
3707static inline unsigned long wait_table_bits(unsigned long size)
3708{
3709 return ffz(~size);
3710}
3711
3712#define LONG_ALIGN(x) (((x)+(sizeof(long))-1)&~((sizeof(long))-1))
3713
Mel Gorman56fd56b2007-10-16 01:25:58 -07003714/*
Arve Hjønnevåg6d3163c2011-05-24 17:12:24 -07003715 * Check if a pageblock contains reserved pages
3716 */
3717static int pageblock_is_reserved(unsigned long start_pfn, unsigned long end_pfn)
3718{
3719 unsigned long pfn;
3720
3721 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
3722 if (!pfn_valid_within(pfn) || PageReserved(pfn_to_page(pfn)))
3723 return 1;
3724 }
3725 return 0;
3726}
3727
3728/*
Mel Gormand9c23402007-10-16 01:26:01 -07003729 * Mark a number of pageblocks as MIGRATE_RESERVE. The number
Mel Gorman41858962009-06-16 15:32:12 -07003730 * of blocks reserved is based on min_wmark_pages(zone). The memory within
3731 * the reserve will tend to store contiguous free pages. Setting min_free_kbytes
Mel Gorman56fd56b2007-10-16 01:25:58 -07003732 * higher will lead to a bigger reserve which will get freed as contiguous
3733 * blocks as reclaim kicks in
3734 */
3735static void setup_zone_migrate_reserve(struct zone *zone)
3736{
Arve Hjønnevåg6d3163c2011-05-24 17:12:24 -07003737 unsigned long start_pfn, pfn, end_pfn, block_end_pfn;
Mel Gorman56fd56b2007-10-16 01:25:58 -07003738 struct page *page;
Mel Gorman78986a62009-09-21 17:03:02 -07003739 unsigned long block_migratetype;
3740 int reserve;
Mel Gorman56fd56b2007-10-16 01:25:58 -07003741
Michal Hockod02156382011-12-08 14:34:27 -08003742 /*
3743 * Get the start pfn, end pfn and the number of blocks to reserve
3744 * We have to be careful to be aligned to pageblock_nr_pages to
3745 * make sure that we always check pfn_valid for the first page in
3746 * the block.
3747 */
Mel Gorman56fd56b2007-10-16 01:25:58 -07003748 start_pfn = zone->zone_start_pfn;
3749 end_pfn = start_pfn + zone->spanned_pages;
Michal Hockod02156382011-12-08 14:34:27 -08003750 start_pfn = roundup(start_pfn, pageblock_nr_pages);
Mel Gorman41858962009-06-16 15:32:12 -07003751 reserve = roundup(min_wmark_pages(zone), pageblock_nr_pages) >>
Mel Gormand9c23402007-10-16 01:26:01 -07003752 pageblock_order;
Mel Gorman56fd56b2007-10-16 01:25:58 -07003753
Mel Gorman78986a62009-09-21 17:03:02 -07003754 /*
3755 * Reserve blocks are generally in place to help high-order atomic
3756 * allocations that are short-lived. A min_free_kbytes value that
3757 * would result in more than 2 reserve blocks for atomic allocations
3758 * is assumed to be in place to help anti-fragmentation for the
3759 * future allocation of hugepages at runtime.
3760 */
3761 reserve = min(2, reserve);
3762
Mel Gormand9c23402007-10-16 01:26:01 -07003763 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
Mel Gorman56fd56b2007-10-16 01:25:58 -07003764 if (!pfn_valid(pfn))
3765 continue;
3766 page = pfn_to_page(pfn);
3767
Adam Litke344c7902008-09-02 14:35:38 -07003768 /* Watch out for overlapping nodes */
3769 if (page_to_nid(page) != zone_to_nid(zone))
3770 continue;
3771
Mel Gorman56fd56b2007-10-16 01:25:58 -07003772 block_migratetype = get_pageblock_migratetype(page);
3773
Mel Gorman938929f2012-01-10 15:07:14 -08003774 /* Only test what is necessary when the reserves are not met */
3775 if (reserve > 0) {
3776 /*
3777 * Blocks with reserved pages will never free, skip
3778 * them.
3779 */
3780 block_end_pfn = min(pfn + pageblock_nr_pages, end_pfn);
3781 if (pageblock_is_reserved(pfn, block_end_pfn))
3782 continue;
Mel Gorman56fd56b2007-10-16 01:25:58 -07003783
Mel Gorman938929f2012-01-10 15:07:14 -08003784 /* If this block is reserved, account for it */
3785 if (block_migratetype == MIGRATE_RESERVE) {
3786 reserve--;
3787 continue;
3788 }
3789
3790 /* Suitable for reserving if this block is movable */
3791 if (block_migratetype == MIGRATE_MOVABLE) {
3792 set_pageblock_migratetype(page,
3793 MIGRATE_RESERVE);
3794 move_freepages_block(zone, page,
3795 MIGRATE_RESERVE);
3796 reserve--;
3797 continue;
3798 }
Mel Gorman56fd56b2007-10-16 01:25:58 -07003799 }
3800
3801 /*
3802 * If the reserve is met and this is a previous reserved block,
3803 * take it back
3804 */
3805 if (block_migratetype == MIGRATE_RESERVE) {
3806 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
3807 move_freepages_block(zone, page, MIGRATE_MOVABLE);
3808 }
3809 }
3810}
Mel Gormanac0e5b72007-10-16 01:25:58 -07003811
Linus Torvalds1da177e2005-04-16 15:20:36 -07003812/*
3813 * Initially all pages are reserved - free ones are freed
3814 * up by free_all_bootmem() once the early boot process is
3815 * done. Non-atomic initialization, single-pass.
3816 */
Matt Tolentinoc09b4242006-01-17 07:03:44 +01003817void __meminit memmap_init_zone(unsigned long size, int nid, unsigned long zone,
Dave Hansena2f3aa02007-01-10 23:15:30 -08003818 unsigned long start_pfn, enum memmap_context context)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003819{
Linus Torvalds1da177e2005-04-16 15:20:36 -07003820 struct page *page;
Andy Whitcroft29751f62005-06-23 00:08:00 -07003821 unsigned long end_pfn = start_pfn + size;
3822 unsigned long pfn;
KAMEZAWA Hiroyuki86051ca2008-04-29 00:58:21 -07003823 struct zone *z;
Linus Torvalds1da177e2005-04-16 15:20:36 -07003824
Hugh Dickins22b31ee2009-01-06 14:40:09 -08003825 if (highest_memmap_pfn < end_pfn - 1)
3826 highest_memmap_pfn = end_pfn - 1;
3827
KAMEZAWA Hiroyuki86051ca2008-04-29 00:58:21 -07003828 z = &NODE_DATA(nid)->node_zones[zone];
Greg Ungerercbe8dd42006-01-12 01:05:24 -08003829 for (pfn = start_pfn; pfn < end_pfn; pfn++) {
Dave Hansena2f3aa02007-01-10 23:15:30 -08003830 /*
3831 * There can be holes in boot-time mem_map[]s
3832 * handed to this function. They do not
3833 * exist on hotplugged memory.
3834 */
3835 if (context == MEMMAP_EARLY) {
3836 if (!early_pfn_valid(pfn))
3837 continue;
3838 if (!early_pfn_in_nid(pfn, nid))
3839 continue;
3840 }
Andy Whitcroftd41dee32005-06-23 00:07:54 -07003841 page = pfn_to_page(pfn);
3842 set_page_links(page, zone, nid, pfn);
Mel Gorman708614e2008-07-23 21:26:51 -07003843 mminit_verify_page_links(page, zone, nid, pfn);
Nick Piggin7835e982006-03-22 00:08:40 -08003844 init_page_count(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003845 reset_page_mapcount(page);
3846 SetPageReserved(page);
Mel Gormanb2a0ac82007-10-16 01:25:48 -07003847 /*
3848 * Mark the block movable so that blocks are reserved for
3849 * movable at startup. This will force kernel allocations
3850 * to reserve their blocks rather than leaking throughout
3851 * the address space during boot when many long-lived
Mel Gorman56fd56b2007-10-16 01:25:58 -07003852 * kernel allocations are made. Later some blocks near
3853 * the start are marked MIGRATE_RESERVE by
3854 * setup_zone_migrate_reserve()
KAMEZAWA Hiroyuki86051ca2008-04-29 00:58:21 -07003855 *
3856 * bitmap is created for zone's valid pfn range. but memmap
3857 * can be created for invalid pages (for alignment)
3858 * check here not to call set_pageblock_migratetype() against
3859 * pfn out of zone.
Mel Gormanb2a0ac82007-10-16 01:25:48 -07003860 */
KAMEZAWA Hiroyuki86051ca2008-04-29 00:58:21 -07003861 if ((z->zone_start_pfn <= pfn)
3862 && (pfn < z->zone_start_pfn + z->spanned_pages)
3863 && !(pfn & (pageblock_nr_pages - 1)))
Mel Gorman56fd56b2007-10-16 01:25:58 -07003864 set_pageblock_migratetype(page, MIGRATE_MOVABLE);
Mel Gormanb2a0ac82007-10-16 01:25:48 -07003865
Linus Torvalds1da177e2005-04-16 15:20:36 -07003866 INIT_LIST_HEAD(&page->lru);
3867#ifdef WANT_PAGE_VIRTUAL
3868 /* The shift won't overflow because ZONE_NORMAL is below 4G. */
3869 if (!is_highmem_idx(zone))
Bob Picco3212c6b2005-06-27 14:36:28 -07003870 set_page_address(page, __va(pfn << PAGE_SHIFT));
Linus Torvalds1da177e2005-04-16 15:20:36 -07003871#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07003872 }
3873}
3874
Andi Kleen1e548de2008-02-04 22:29:26 -08003875static void __meminit zone_init_free_lists(struct zone *zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003876{
Mel Gormanb2a0ac82007-10-16 01:25:48 -07003877 int order, t;
3878 for_each_migratetype_order(order, t) {
3879 INIT_LIST_HEAD(&zone->free_area[order].free_list[t]);
Linus Torvalds1da177e2005-04-16 15:20:36 -07003880 zone->free_area[order].nr_free = 0;
3881 }
3882}
3883
3884#ifndef __HAVE_ARCH_MEMMAP_INIT
3885#define memmap_init(size, nid, zone, start_pfn) \
Dave Hansena2f3aa02007-01-10 23:15:30 -08003886 memmap_init_zone((size), (nid), (zone), (start_pfn), MEMMAP_EARLY)
Linus Torvalds1da177e2005-04-16 15:20:36 -07003887#endif
3888
Sam Ravnborg1d6f4e62008-01-17 15:21:12 -08003889static int zone_batchsize(struct zone *zone)
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07003890{
David Howells3a6be872009-05-06 16:03:03 -07003891#ifdef CONFIG_MMU
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07003892 int batch;
3893
3894 /*
3895 * The per-cpu-pages pools are set to around 1000th of the
Seth, Rohitba56e912005-10-29 18:15:47 -07003896 * size of the zone. But no more than 1/2 of a meg.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07003897 *
3898 * OK, so we don't know how big the cache is. So guess.
3899 */
3900 batch = zone->present_pages / 1024;
Seth, Rohitba56e912005-10-29 18:15:47 -07003901 if (batch * PAGE_SIZE > 512 * 1024)
3902 batch = (512 * 1024) / PAGE_SIZE;
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07003903 batch /= 4; /* We effectively *= 4 below */
3904 if (batch < 1)
3905 batch = 1;
3906
3907 /*
Nick Piggin0ceaacc2005-12-04 13:55:25 +11003908 * Clamp the batch to a 2^n - 1 value. Having a power
3909 * of 2 value was found to be more likely to have
3910 * suboptimal cache aliasing properties in some cases.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07003911 *
Nick Piggin0ceaacc2005-12-04 13:55:25 +11003912 * For example if 2 tasks are alternately allocating
3913 * batches of pages, one task can end up with a lot
3914 * of pages of one half of the possible page colors
3915 * and the other with pages of the other colors.
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07003916 */
David Howells91552032009-05-06 16:03:02 -07003917 batch = rounddown_pow_of_two(batch + batch/2) - 1;
Seth, Rohitba56e912005-10-29 18:15:47 -07003918
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07003919 return batch;
David Howells3a6be872009-05-06 16:03:03 -07003920
3921#else
3922 /* The deferral and batching of frees should be suppressed under NOMMU
3923 * conditions.
3924 *
3925 * The problem is that NOMMU needs to be able to allocate large chunks
3926 * of contiguous memory as there's no hardware page translation to
3927 * assemble apparent contiguous memory from discontiguous pages.
3928 *
3929 * Queueing large contiguous runs of pages for batching, however,
3930 * causes the pages to actually be freed in smaller chunks. As there
3931 * can be a significant delay between the individual batches being
3932 * recycled, this leads to the once large chunks of space being
3933 * fragmented and becoming unavailable for high-order allocations.
3934 */
3935 return 0;
3936#endif
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07003937}
3938
Adrian Bunkb69a7282008-07-23 21:28:12 -07003939static void setup_pageset(struct per_cpu_pageset *p, unsigned long batch)
Christoph Lameter2caaad42005-06-21 17:15:00 -07003940{
3941 struct per_cpu_pages *pcp;
Mel Gorman5f8dcc22009-09-21 17:03:19 -07003942 int migratetype;
Christoph Lameter2caaad42005-06-21 17:15:00 -07003943
Magnus Damm1c6fe942005-10-26 01:58:59 -07003944 memset(p, 0, sizeof(*p));
3945
Christoph Lameter3dfa5722008-02-04 22:29:19 -08003946 pcp = &p->pcp;
Christoph Lameter2caaad42005-06-21 17:15:00 -07003947 pcp->count = 0;
Christoph Lameter2caaad42005-06-21 17:15:00 -07003948 pcp->high = 6 * batch;
3949 pcp->batch = max(1UL, 1 * batch);
Mel Gorman5f8dcc22009-09-21 17:03:19 -07003950 for (migratetype = 0; migratetype < MIGRATE_PCPTYPES; migratetype++)
3951 INIT_LIST_HEAD(&pcp->lists[migratetype]);
Christoph Lameter2caaad42005-06-21 17:15:00 -07003952}
3953
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08003954/*
3955 * setup_pagelist_highmark() sets the high water mark for hot per_cpu_pagelist
3956 * to the value high for the pageset p.
3957 */
3958
3959static void setup_pagelist_highmark(struct per_cpu_pageset *p,
3960 unsigned long high)
3961{
3962 struct per_cpu_pages *pcp;
3963
Christoph Lameter3dfa5722008-02-04 22:29:19 -08003964 pcp = &p->pcp;
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08003965 pcp->high = high;
3966 pcp->batch = max(1UL, high/4);
3967 if ((high/4) > (PAGE_SHIFT * 8))
3968 pcp->batch = PAGE_SHIFT * 8;
3969}
3970
Nikanth Karthikesan58c2ee42011-03-15 10:59:02 +05303971static void setup_zone_pageset(struct zone *zone)
Wu Fengguang319774e2010-05-24 14:32:49 -07003972{
3973 int cpu;
3974
3975 zone->pageset = alloc_percpu(struct per_cpu_pageset);
3976
3977 for_each_possible_cpu(cpu) {
3978 struct per_cpu_pageset *pcp = per_cpu_ptr(zone->pageset, cpu);
3979
3980 setup_pageset(pcp, zone_batchsize(zone));
3981
3982 if (percpu_pagelist_fraction)
3983 setup_pagelist_highmark(pcp,
3984 (zone->present_pages /
3985 percpu_pagelist_fraction));
3986 }
3987}
3988
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07003989/*
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09003990 * Allocate per cpu pagesets and initialize them.
3991 * Before this call only boot pagesets were available.
Christoph Lameter2caaad42005-06-21 17:15:00 -07003992 */
Al Viro78d99552005-12-15 09:18:25 +00003993void __init setup_per_cpu_pageset(void)
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07003994{
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09003995 struct zone *zone;
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07003996
Wu Fengguang319774e2010-05-24 14:32:49 -07003997 for_each_populated_zone(zone)
3998 setup_zone_pageset(zone);
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07003999}
4000
Sam Ravnborg577a32f2007-05-17 23:29:25 +02004001static noinline __init_refok
Yasunori Gotocca448f2006-06-23 02:03:10 -07004002int zone_wait_table_init(struct zone *zone, unsigned long zone_size_pages)
Dave Hansened8ece22005-10-29 18:16:50 -07004003{
4004 int i;
4005 struct pglist_data *pgdat = zone->zone_pgdat;
Yasunori Gotocca448f2006-06-23 02:03:10 -07004006 size_t alloc_size;
Dave Hansened8ece22005-10-29 18:16:50 -07004007
4008 /*
4009 * The per-page waitqueue mechanism uses hashed waitqueues
4010 * per zone.
4011 */
Yasunori Goto02b694d2006-06-23 02:03:08 -07004012 zone->wait_table_hash_nr_entries =
4013 wait_table_hash_nr_entries(zone_size_pages);
4014 zone->wait_table_bits =
4015 wait_table_bits(zone->wait_table_hash_nr_entries);
Yasunori Gotocca448f2006-06-23 02:03:10 -07004016 alloc_size = zone->wait_table_hash_nr_entries
4017 * sizeof(wait_queue_head_t);
4018
Heiko Carstenscd94b9d2008-05-23 13:04:52 -07004019 if (!slab_is_available()) {
Yasunori Gotocca448f2006-06-23 02:03:10 -07004020 zone->wait_table = (wait_queue_head_t *)
Yinghai Lu8f389a992011-05-11 15:13:32 -07004021 alloc_bootmem_node_nopanic(pgdat, alloc_size);
Yasunori Gotocca448f2006-06-23 02:03:10 -07004022 } else {
4023 /*
4024 * This case means that a zone whose size was 0 gets new memory
4025 * via memory hot-add.
4026 * But it may be the case that a new node was hot-added. In
4027 * this case vmalloc() will not be able to use this new node's
4028 * memory - this wait_table must be initialized to use this new
4029 * node itself as well.
4030 * To use this new node's memory, further consideration will be
4031 * necessary.
4032 */
Jesper Juhl8691f3a2007-10-16 01:24:49 -07004033 zone->wait_table = vmalloc(alloc_size);
Yasunori Gotocca448f2006-06-23 02:03:10 -07004034 }
4035 if (!zone->wait_table)
4036 return -ENOMEM;
Dave Hansened8ece22005-10-29 18:16:50 -07004037
Yasunori Goto02b694d2006-06-23 02:03:08 -07004038 for(i = 0; i < zone->wait_table_hash_nr_entries; ++i)
Dave Hansened8ece22005-10-29 18:16:50 -07004039 init_waitqueue_head(zone->wait_table + i);
Yasunori Gotocca448f2006-06-23 02:03:10 -07004040
4041 return 0;
Dave Hansened8ece22005-10-29 18:16:50 -07004042}
4043
Shaohua Li112067f2009-09-21 17:01:16 -07004044static int __zone_pcp_update(void *data)
4045{
4046 struct zone *zone = data;
4047 int cpu;
4048 unsigned long batch = zone_batchsize(zone), flags;
4049
Thomas Gleixner2d30a1f2010-03-10 15:20:40 -08004050 for_each_possible_cpu(cpu) {
Shaohua Li112067f2009-09-21 17:01:16 -07004051 struct per_cpu_pageset *pset;
4052 struct per_cpu_pages *pcp;
4053
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09004054 pset = per_cpu_ptr(zone->pageset, cpu);
Shaohua Li112067f2009-09-21 17:01:16 -07004055 pcp = &pset->pcp;
4056
4057 local_irq_save(flags);
Mel Gorman5f8dcc22009-09-21 17:03:19 -07004058 free_pcppages_bulk(zone, pcp->count, pcp);
Shaohua Li112067f2009-09-21 17:01:16 -07004059 setup_pageset(pset, batch);
4060 local_irq_restore(flags);
4061 }
4062 return 0;
4063}
4064
4065void zone_pcp_update(struct zone *zone)
4066{
4067 stop_machine(__zone_pcp_update, zone, NULL);
4068}
4069
Matt Tolentinoc09b4242006-01-17 07:03:44 +01004070static __meminit void zone_pcp_init(struct zone *zone)
Dave Hansened8ece22005-10-29 18:16:50 -07004071{
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09004072 /*
4073 * per cpu subsystem is not up at this point. The following code
4074 * relies on the ability of the linker to provide the
4075 * offset of a (static) per cpu variable into the per cpu area.
4076 */
4077 zone->pageset = &boot_pageset;
Dave Hansened8ece22005-10-29 18:16:50 -07004078
Anton Blanchardf5335c02006-03-25 03:06:49 -08004079 if (zone->present_pages)
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09004080 printk(KERN_DEBUG " %s zone: %lu pages, LIFO batch:%u\n",
4081 zone->name, zone->present_pages,
4082 zone_batchsize(zone));
Dave Hansened8ece22005-10-29 18:16:50 -07004083}
4084
Yasunori Goto718127c2006-06-23 02:03:10 -07004085__meminit int init_currently_empty_zone(struct zone *zone,
4086 unsigned long zone_start_pfn,
Dave Hansena2f3aa02007-01-10 23:15:30 -08004087 unsigned long size,
4088 enum memmap_context context)
Dave Hansened8ece22005-10-29 18:16:50 -07004089{
4090 struct pglist_data *pgdat = zone->zone_pgdat;
Yasunori Gotocca448f2006-06-23 02:03:10 -07004091 int ret;
4092 ret = zone_wait_table_init(zone, size);
4093 if (ret)
4094 return ret;
Dave Hansened8ece22005-10-29 18:16:50 -07004095 pgdat->nr_zones = zone_idx(zone) + 1;
4096
Dave Hansened8ece22005-10-29 18:16:50 -07004097 zone->zone_start_pfn = zone_start_pfn;
4098
Mel Gorman708614e2008-07-23 21:26:51 -07004099 mminit_dprintk(MMINIT_TRACE, "memmap_init",
4100 "Initialising map node %d zone %lu pfns %lu -> %lu\n",
4101 pgdat->node_id,
4102 (unsigned long)zone_idx(zone),
4103 zone_start_pfn, (zone_start_pfn + size));
4104
Andi Kleen1e548de2008-02-04 22:29:26 -08004105 zone_init_free_lists(zone);
Yasunori Goto718127c2006-06-23 02:03:10 -07004106
4107 return 0;
Dave Hansened8ece22005-10-29 18:16:50 -07004108}
4109
Tejun Heo0ee332c2011-12-08 10:22:09 -08004110#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
Mel Gormanc7132162006-09-27 01:49:43 -07004111#ifndef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
4112/*
4113 * Required by SPARSEMEM. Given a PFN, return what node the PFN is on.
4114 * Architectures may implement their own version but if add_active_range()
4115 * was used and there are no special requirements, this is a convenient
4116 * alternative
4117 */
KAMEZAWA Hiroyukif2dbcfa2009-02-18 14:48:32 -08004118int __meminit __early_pfn_to_nid(unsigned long pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07004119{
Tejun Heoc13291a2011-07-12 10:46:30 +02004120 unsigned long start_pfn, end_pfn;
4121 int i, nid;
Mel Gormanc7132162006-09-27 01:49:43 -07004122
Tejun Heoc13291a2011-07-12 10:46:30 +02004123 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
Mel Gormanc7132162006-09-27 01:49:43 -07004124 if (start_pfn <= pfn && pfn < end_pfn)
Tejun Heoc13291a2011-07-12 10:46:30 +02004125 return nid;
KAMEZAWA Hiroyukicc2559b2009-02-18 14:48:33 -08004126 /* This is a memory hole */
4127 return -1;
Mel Gormanc7132162006-09-27 01:49:43 -07004128}
4129#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
4130
KAMEZAWA Hiroyukif2dbcfa2009-02-18 14:48:32 -08004131int __meminit early_pfn_to_nid(unsigned long pfn)
4132{
KAMEZAWA Hiroyukicc2559b2009-02-18 14:48:33 -08004133 int nid;
4134
4135 nid = __early_pfn_to_nid(pfn);
4136 if (nid >= 0)
4137 return nid;
4138 /* just returns 0 */
4139 return 0;
KAMEZAWA Hiroyukif2dbcfa2009-02-18 14:48:32 -08004140}
4141
KAMEZAWA Hiroyukicc2559b2009-02-18 14:48:33 -08004142#ifdef CONFIG_NODES_SPAN_OTHER_NODES
4143bool __meminit early_pfn_in_nid(unsigned long pfn, int node)
4144{
4145 int nid;
4146
4147 nid = __early_pfn_to_nid(pfn);
4148 if (nid >= 0 && nid != node)
4149 return false;
4150 return true;
4151}
4152#endif
KAMEZAWA Hiroyukif2dbcfa2009-02-18 14:48:32 -08004153
Mel Gormanc7132162006-09-27 01:49:43 -07004154/**
4155 * free_bootmem_with_active_regions - Call free_bootmem_node for each active range
Randy Dunlap88ca3b92006-10-04 02:15:25 -07004156 * @nid: The node to free memory on. If MAX_NUMNODES, all nodes are freed.
4157 * @max_low_pfn: The highest PFN that will be passed to free_bootmem_node
Mel Gormanc7132162006-09-27 01:49:43 -07004158 *
4159 * If an architecture guarantees that all ranges registered with
4160 * add_active_ranges() contain no holes and may be freed, this
4161 * this function may be used instead of calling free_bootmem() manually.
4162 */
Tejun Heoc13291a2011-07-12 10:46:30 +02004163void __init free_bootmem_with_active_regions(int nid, unsigned long max_low_pfn)
Mel Gormanc7132162006-09-27 01:49:43 -07004164{
Tejun Heoc13291a2011-07-12 10:46:30 +02004165 unsigned long start_pfn, end_pfn;
4166 int i, this_nid;
Mel Gormanc7132162006-09-27 01:49:43 -07004167
Tejun Heoc13291a2011-07-12 10:46:30 +02004168 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid) {
4169 start_pfn = min(start_pfn, max_low_pfn);
4170 end_pfn = min(end_pfn, max_low_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07004171
Tejun Heoc13291a2011-07-12 10:46:30 +02004172 if (start_pfn < end_pfn)
4173 free_bootmem_node(NODE_DATA(this_nid),
4174 PFN_PHYS(start_pfn),
4175 (end_pfn - start_pfn) << PAGE_SHIFT);
Mel Gormanc7132162006-09-27 01:49:43 -07004176 }
4177}
4178
4179/**
4180 * sparse_memory_present_with_active_regions - Call memory_present for each active range
Randy Dunlap88ca3b92006-10-04 02:15:25 -07004181 * @nid: The node to call memory_present for. If MAX_NUMNODES, all nodes will be used.
Mel Gormanc7132162006-09-27 01:49:43 -07004182 *
4183 * If an architecture guarantees that all ranges registered with
4184 * add_active_ranges() contain no holes and may be freed, this
Randy Dunlap88ca3b92006-10-04 02:15:25 -07004185 * function may be used instead of calling memory_present() manually.
Mel Gormanc7132162006-09-27 01:49:43 -07004186 */
4187void __init sparse_memory_present_with_active_regions(int nid)
4188{
Tejun Heoc13291a2011-07-12 10:46:30 +02004189 unsigned long start_pfn, end_pfn;
4190 int i, this_nid;
Mel Gormanc7132162006-09-27 01:49:43 -07004191
Tejun Heoc13291a2011-07-12 10:46:30 +02004192 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, &this_nid)
4193 memory_present(this_nid, start_pfn, end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07004194}
4195
4196/**
4197 * get_pfn_range_for_nid - Return the start and end page frames for a node
Randy Dunlap88ca3b92006-10-04 02:15:25 -07004198 * @nid: The nid to return the range for. If MAX_NUMNODES, the min and max PFN are returned.
4199 * @start_pfn: Passed by reference. On return, it will have the node start_pfn.
4200 * @end_pfn: Passed by reference. On return, it will have the node end_pfn.
Mel Gormanc7132162006-09-27 01:49:43 -07004201 *
4202 * It returns the start and end page frame of a node based on information
4203 * provided by an arch calling add_active_range(). If called for a node
4204 * with no available memory, a warning is printed and the start and end
Randy Dunlap88ca3b92006-10-04 02:15:25 -07004205 * PFNs will be 0.
Mel Gormanc7132162006-09-27 01:49:43 -07004206 */
Yasunori Gotoa3142c82007-05-08 00:23:07 -07004207void __meminit get_pfn_range_for_nid(unsigned int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07004208 unsigned long *start_pfn, unsigned long *end_pfn)
4209{
Tejun Heoc13291a2011-07-12 10:46:30 +02004210 unsigned long this_start_pfn, this_end_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07004211 int i;
Tejun Heoc13291a2011-07-12 10:46:30 +02004212
Mel Gormanc7132162006-09-27 01:49:43 -07004213 *start_pfn = -1UL;
4214 *end_pfn = 0;
4215
Tejun Heoc13291a2011-07-12 10:46:30 +02004216 for_each_mem_pfn_range(i, nid, &this_start_pfn, &this_end_pfn, NULL) {
4217 *start_pfn = min(*start_pfn, this_start_pfn);
4218 *end_pfn = max(*end_pfn, this_end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07004219 }
4220
Christoph Lameter633c0662007-10-16 01:25:37 -07004221 if (*start_pfn == -1UL)
Mel Gormanc7132162006-09-27 01:49:43 -07004222 *start_pfn = 0;
Mel Gormanc7132162006-09-27 01:49:43 -07004223}
4224
4225/*
Mel Gorman2a1e2742007-07-17 04:03:12 -07004226 * This finds a zone that can be used for ZONE_MOVABLE pages. The
4227 * assumption is made that zones within a node are ordered in monotonic
4228 * increasing memory addresses so that the "highest" populated zone is used
4229 */
Adrian Bunkb69a7282008-07-23 21:28:12 -07004230static void __init find_usable_zone_for_movable(void)
Mel Gorman2a1e2742007-07-17 04:03:12 -07004231{
4232 int zone_index;
4233 for (zone_index = MAX_NR_ZONES - 1; zone_index >= 0; zone_index--) {
4234 if (zone_index == ZONE_MOVABLE)
4235 continue;
4236
4237 if (arch_zone_highest_possible_pfn[zone_index] >
4238 arch_zone_lowest_possible_pfn[zone_index])
4239 break;
4240 }
4241
4242 VM_BUG_ON(zone_index == -1);
4243 movable_zone = zone_index;
4244}
4245
4246/*
4247 * The zone ranges provided by the architecture do not include ZONE_MOVABLE
Lucas De Marchi25985ed2011-03-30 22:57:33 -03004248 * because it is sized independent of architecture. Unlike the other zones,
Mel Gorman2a1e2742007-07-17 04:03:12 -07004249 * the starting point for ZONE_MOVABLE is not fixed. It may be different
4250 * in each node depending on the size of each node and how evenly kernelcore
4251 * is distributed. This helper function adjusts the zone ranges
4252 * provided by the architecture for a given node by using the end of the
4253 * highest usable zone for ZONE_MOVABLE. This preserves the assumption that
4254 * zones within a node are in order of monotonic increases memory addresses
4255 */
Adrian Bunkb69a7282008-07-23 21:28:12 -07004256static void __meminit adjust_zone_range_for_zone_movable(int nid,
Mel Gorman2a1e2742007-07-17 04:03:12 -07004257 unsigned long zone_type,
4258 unsigned long node_start_pfn,
4259 unsigned long node_end_pfn,
4260 unsigned long *zone_start_pfn,
4261 unsigned long *zone_end_pfn)
4262{
4263 /* Only adjust if ZONE_MOVABLE is on this node */
4264 if (zone_movable_pfn[nid]) {
4265 /* Size ZONE_MOVABLE */
4266 if (zone_type == ZONE_MOVABLE) {
4267 *zone_start_pfn = zone_movable_pfn[nid];
4268 *zone_end_pfn = min(node_end_pfn,
4269 arch_zone_highest_possible_pfn[movable_zone]);
4270
4271 /* Adjust for ZONE_MOVABLE starting within this range */
4272 } else if (*zone_start_pfn < zone_movable_pfn[nid] &&
4273 *zone_end_pfn > zone_movable_pfn[nid]) {
4274 *zone_end_pfn = zone_movable_pfn[nid];
4275
4276 /* Check if this whole range is within ZONE_MOVABLE */
4277 } else if (*zone_start_pfn >= zone_movable_pfn[nid])
4278 *zone_start_pfn = *zone_end_pfn;
4279 }
4280}
4281
4282/*
Mel Gormanc7132162006-09-27 01:49:43 -07004283 * Return the number of pages a zone spans in a node, including holes
4284 * present_pages = zone_spanned_pages_in_node() - zone_absent_pages_in_node()
4285 */
Paul Mundt6ea6e682007-07-15 23:38:20 -07004286static unsigned long __meminit zone_spanned_pages_in_node(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07004287 unsigned long zone_type,
4288 unsigned long *ignored)
4289{
4290 unsigned long node_start_pfn, node_end_pfn;
4291 unsigned long zone_start_pfn, zone_end_pfn;
4292
4293 /* Get the start and end of the node and zone */
4294 get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
4295 zone_start_pfn = arch_zone_lowest_possible_pfn[zone_type];
4296 zone_end_pfn = arch_zone_highest_possible_pfn[zone_type];
Mel Gorman2a1e2742007-07-17 04:03:12 -07004297 adjust_zone_range_for_zone_movable(nid, zone_type,
4298 node_start_pfn, node_end_pfn,
4299 &zone_start_pfn, &zone_end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07004300
4301 /* Check that this node has pages within the zone's required range */
4302 if (zone_end_pfn < node_start_pfn || zone_start_pfn > node_end_pfn)
4303 return 0;
4304
4305 /* Move the zone boundaries inside the node if necessary */
4306 zone_end_pfn = min(zone_end_pfn, node_end_pfn);
4307 zone_start_pfn = max(zone_start_pfn, node_start_pfn);
4308
4309 /* Return the spanned pages */
4310 return zone_end_pfn - zone_start_pfn;
4311}
4312
4313/*
4314 * Return the number of holes in a range on a node. If nid is MAX_NUMNODES,
Randy Dunlap88ca3b92006-10-04 02:15:25 -07004315 * then all holes in the requested range will be accounted for.
Mel Gormanc7132162006-09-27 01:49:43 -07004316 */
Yinghai Lu32996252009-12-15 17:59:02 -08004317unsigned long __meminit __absent_pages_in_range(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07004318 unsigned long range_start_pfn,
4319 unsigned long range_end_pfn)
4320{
Tejun Heo96e907d2011-07-12 10:46:29 +02004321 unsigned long nr_absent = range_end_pfn - range_start_pfn;
4322 unsigned long start_pfn, end_pfn;
4323 int i;
Mel Gormanc7132162006-09-27 01:49:43 -07004324
Tejun Heo96e907d2011-07-12 10:46:29 +02004325 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
4326 start_pfn = clamp(start_pfn, range_start_pfn, range_end_pfn);
4327 end_pfn = clamp(end_pfn, range_start_pfn, range_end_pfn);
4328 nr_absent -= end_pfn - start_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07004329 }
Tejun Heo96e907d2011-07-12 10:46:29 +02004330 return nr_absent;
Mel Gormanc7132162006-09-27 01:49:43 -07004331}
4332
4333/**
4334 * absent_pages_in_range - Return number of page frames in holes within a range
4335 * @start_pfn: The start PFN to start searching for holes
4336 * @end_pfn: The end PFN to stop searching for holes
4337 *
Randy Dunlap88ca3b92006-10-04 02:15:25 -07004338 * It returns the number of pages frames in memory holes within a range.
Mel Gormanc7132162006-09-27 01:49:43 -07004339 */
4340unsigned long __init absent_pages_in_range(unsigned long start_pfn,
4341 unsigned long end_pfn)
4342{
4343 return __absent_pages_in_range(MAX_NUMNODES, start_pfn, end_pfn);
4344}
4345
4346/* Return the number of page frames in holes in a zone on a node */
Paul Mundt6ea6e682007-07-15 23:38:20 -07004347static unsigned long __meminit zone_absent_pages_in_node(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07004348 unsigned long zone_type,
4349 unsigned long *ignored)
4350{
Tejun Heo96e907d2011-07-12 10:46:29 +02004351 unsigned long zone_low = arch_zone_lowest_possible_pfn[zone_type];
4352 unsigned long zone_high = arch_zone_highest_possible_pfn[zone_type];
Mel Gorman9c7cd682006-09-27 01:49:58 -07004353 unsigned long node_start_pfn, node_end_pfn;
4354 unsigned long zone_start_pfn, zone_end_pfn;
4355
4356 get_pfn_range_for_nid(nid, &node_start_pfn, &node_end_pfn);
Tejun Heo96e907d2011-07-12 10:46:29 +02004357 zone_start_pfn = clamp(node_start_pfn, zone_low, zone_high);
4358 zone_end_pfn = clamp(node_end_pfn, zone_low, zone_high);
Mel Gorman9c7cd682006-09-27 01:49:58 -07004359
Mel Gorman2a1e2742007-07-17 04:03:12 -07004360 adjust_zone_range_for_zone_movable(nid, zone_type,
4361 node_start_pfn, node_end_pfn,
4362 &zone_start_pfn, &zone_end_pfn);
Mel Gorman9c7cd682006-09-27 01:49:58 -07004363 return __absent_pages_in_range(nid, zone_start_pfn, zone_end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07004364}
Mel Gorman0e0b8642006-09-27 01:49:56 -07004365
Tejun Heo0ee332c2011-12-08 10:22:09 -08004366#else /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
Paul Mundt6ea6e682007-07-15 23:38:20 -07004367static inline unsigned long __meminit zone_spanned_pages_in_node(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07004368 unsigned long zone_type,
4369 unsigned long *zones_size)
4370{
4371 return zones_size[zone_type];
4372}
4373
Paul Mundt6ea6e682007-07-15 23:38:20 -07004374static inline unsigned long __meminit zone_absent_pages_in_node(int nid,
Mel Gormanc7132162006-09-27 01:49:43 -07004375 unsigned long zone_type,
4376 unsigned long *zholes_size)
4377{
4378 if (!zholes_size)
4379 return 0;
4380
4381 return zholes_size[zone_type];
4382}
Mel Gorman0e0b8642006-09-27 01:49:56 -07004383
Tejun Heo0ee332c2011-12-08 10:22:09 -08004384#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
Mel Gormanc7132162006-09-27 01:49:43 -07004385
Yasunori Gotoa3142c82007-05-08 00:23:07 -07004386static void __meminit calculate_node_totalpages(struct pglist_data *pgdat,
Mel Gormanc7132162006-09-27 01:49:43 -07004387 unsigned long *zones_size, unsigned long *zholes_size)
4388{
4389 unsigned long realtotalpages, totalpages = 0;
4390 enum zone_type i;
4391
4392 for (i = 0; i < MAX_NR_ZONES; i++)
4393 totalpages += zone_spanned_pages_in_node(pgdat->node_id, i,
4394 zones_size);
4395 pgdat->node_spanned_pages = totalpages;
4396
4397 realtotalpages = totalpages;
4398 for (i = 0; i < MAX_NR_ZONES; i++)
4399 realtotalpages -=
4400 zone_absent_pages_in_node(pgdat->node_id, i,
4401 zholes_size);
4402 pgdat->node_present_pages = realtotalpages;
4403 printk(KERN_DEBUG "On node %d totalpages: %lu\n", pgdat->node_id,
4404 realtotalpages);
4405}
4406
Mel Gorman835c1342007-10-16 01:25:47 -07004407#ifndef CONFIG_SPARSEMEM
4408/*
4409 * Calculate the size of the zone->blockflags rounded to an unsigned long
Mel Gormand9c23402007-10-16 01:26:01 -07004410 * Start by making sure zonesize is a multiple of pageblock_order by rounding
4411 * up. Then use 1 NR_PAGEBLOCK_BITS worth of bits per pageblock, finally
Mel Gorman835c1342007-10-16 01:25:47 -07004412 * round what is now in bits to nearest long in bits, then return it in
4413 * bytes.
4414 */
4415static unsigned long __init usemap_size(unsigned long zonesize)
4416{
4417 unsigned long usemapsize;
4418
Mel Gormand9c23402007-10-16 01:26:01 -07004419 usemapsize = roundup(zonesize, pageblock_nr_pages);
4420 usemapsize = usemapsize >> pageblock_order;
Mel Gorman835c1342007-10-16 01:25:47 -07004421 usemapsize *= NR_PAGEBLOCK_BITS;
4422 usemapsize = roundup(usemapsize, 8 * sizeof(unsigned long));
4423
4424 return usemapsize / 8;
4425}
4426
4427static void __init setup_usemap(struct pglist_data *pgdat,
4428 struct zone *zone, unsigned long zonesize)
4429{
4430 unsigned long usemapsize = usemap_size(zonesize);
4431 zone->pageblock_flags = NULL;
Julia Lawall58a01a42009-01-06 14:39:28 -08004432 if (usemapsize)
Yinghai Lu8f389a992011-05-11 15:13:32 -07004433 zone->pageblock_flags = alloc_bootmem_node_nopanic(pgdat,
4434 usemapsize);
Mel Gorman835c1342007-10-16 01:25:47 -07004435}
4436#else
Jesper Juhlfa9f90b2010-11-28 21:39:34 +01004437static inline void setup_usemap(struct pglist_data *pgdat,
Mel Gorman835c1342007-10-16 01:25:47 -07004438 struct zone *zone, unsigned long zonesize) {}
4439#endif /* CONFIG_SPARSEMEM */
4440
Mel Gormand9c23402007-10-16 01:26:01 -07004441#ifdef CONFIG_HUGETLB_PAGE_SIZE_VARIABLE
Mel Gormanba72cb82007-11-28 16:21:13 -08004442
4443/* Return a sensible default order for the pageblock size. */
4444static inline int pageblock_default_order(void)
4445{
4446 if (HPAGE_SHIFT > PAGE_SHIFT)
4447 return HUGETLB_PAGE_ORDER;
4448
4449 return MAX_ORDER-1;
4450}
4451
Mel Gormand9c23402007-10-16 01:26:01 -07004452/* Initialise the number of pages represented by NR_PAGEBLOCK_BITS */
4453static inline void __init set_pageblock_order(unsigned int order)
4454{
4455 /* Check that pageblock_nr_pages has not already been setup */
4456 if (pageblock_order)
4457 return;
4458
4459 /*
4460 * Assume the largest contiguous order of interest is a huge page.
4461 * This value may be variable depending on boot parameters on IA64
4462 */
4463 pageblock_order = order;
4464}
4465#else /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
4466
Mel Gormanba72cb82007-11-28 16:21:13 -08004467/*
4468 * When CONFIG_HUGETLB_PAGE_SIZE_VARIABLE is not set, set_pageblock_order()
4469 * and pageblock_default_order() are unused as pageblock_order is set
4470 * at compile-time. See include/linux/pageblock-flags.h for the values of
4471 * pageblock_order based on the kernel config
4472 */
4473static inline int pageblock_default_order(unsigned int order)
4474{
4475 return MAX_ORDER-1;
4476}
Mel Gormand9c23402007-10-16 01:26:01 -07004477#define set_pageblock_order(x) do {} while (0)
4478
4479#endif /* CONFIG_HUGETLB_PAGE_SIZE_VARIABLE */
4480
Linus Torvalds1da177e2005-04-16 15:20:36 -07004481/*
4482 * Set up the zone data structures:
4483 * - mark all pages reserved
4484 * - mark all memory queues empty
4485 * - clear the memory bitmaps
4486 */
Alexander van Heukelumb5a0e012008-02-23 15:24:06 -08004487static void __paginginit free_area_init_core(struct pglist_data *pgdat,
Linus Torvalds1da177e2005-04-16 15:20:36 -07004488 unsigned long *zones_size, unsigned long *zholes_size)
4489{
Christoph Lameter2f1b6242006-09-25 23:31:13 -07004490 enum zone_type j;
Dave Hansened8ece22005-10-29 18:16:50 -07004491 int nid = pgdat->node_id;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004492 unsigned long zone_start_pfn = pgdat->node_start_pfn;
Yasunori Goto718127c2006-06-23 02:03:10 -07004493 int ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004494
Dave Hansen208d54e2005-10-29 18:16:52 -07004495 pgdat_resize_init(pgdat);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004496 pgdat->nr_zones = 0;
4497 init_waitqueue_head(&pgdat->kswapd_wait);
4498 pgdat->kswapd_max_order = 0;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -07004499 pgdat_page_cgroup_init(pgdat);
Michal Nazarewiczadc95fd2012-01-11 15:16:11 +01004500
Linus Torvalds1da177e2005-04-16 15:20:36 -07004501 for (j = 0; j < MAX_NR_ZONES; j++) {
4502 struct zone *zone = pgdat->node_zones + j;
Mel Gorman0e0b8642006-09-27 01:49:56 -07004503 unsigned long size, realsize, memmap_pages;
Hugh Dickins41113042012-01-12 17:20:01 -08004504 enum lru_list lru;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004505
Mel Gormanc7132162006-09-27 01:49:43 -07004506 size = zone_spanned_pages_in_node(nid, j, zones_size);
4507 realsize = size - zone_absent_pages_in_node(nid, j,
4508 zholes_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004509
Mel Gorman0e0b8642006-09-27 01:49:56 -07004510 /*
4511 * Adjust realsize so that it accounts for how much memory
4512 * is used by this zone for memmap. This affects the watermark
4513 * and per-cpu initialisations
4514 */
Johannes Weinerf7232152008-05-23 13:04:21 -07004515 memmap_pages =
4516 PAGE_ALIGN(size * sizeof(struct page)) >> PAGE_SHIFT;
Mel Gorman0e0b8642006-09-27 01:49:56 -07004517 if (realsize >= memmap_pages) {
4518 realsize -= memmap_pages;
Yinghai Lu5594c8c2009-01-06 14:39:14 -08004519 if (memmap_pages)
4520 printk(KERN_DEBUG
4521 " %s zone: %lu pages used for memmap\n",
4522 zone_names[j], memmap_pages);
Mel Gorman0e0b8642006-09-27 01:49:56 -07004523 } else
4524 printk(KERN_WARNING
4525 " %s zone: %lu pages exceeds realsize %lu\n",
4526 zone_names[j], memmap_pages, realsize);
4527
Christoph Lameter62672762007-02-10 01:43:07 -08004528 /* Account for reserved pages */
4529 if (j == 0 && realsize > dma_reserve) {
Mel Gorman0e0b8642006-09-27 01:49:56 -07004530 realsize -= dma_reserve;
Yinghai Lud903ef92008-10-18 20:27:06 -07004531 printk(KERN_DEBUG " %s zone: %lu pages reserved\n",
Christoph Lameter62672762007-02-10 01:43:07 -08004532 zone_names[0], dma_reserve);
Mel Gorman0e0b8642006-09-27 01:49:56 -07004533 }
4534
Christoph Lameter98d2b0e2006-09-25 23:31:12 -07004535 if (!is_highmem_idx(j))
Linus Torvalds1da177e2005-04-16 15:20:36 -07004536 nr_kernel_pages += realsize;
4537 nr_all_pages += realsize;
4538
4539 zone->spanned_pages = size;
4540 zone->present_pages = realsize;
Christoph Lameter96146342006-07-03 00:24:13 -07004541#ifdef CONFIG_NUMA
Christoph Lameterd5f541e2006-09-27 01:50:08 -07004542 zone->node = nid;
Christoph Lameter8417bba2006-09-25 23:31:51 -07004543 zone->min_unmapped_pages = (realsize*sysctl_min_unmapped_ratio)
Christoph Lameter96146342006-07-03 00:24:13 -07004544 / 100;
Christoph Lameter0ff38492006-09-25 23:31:52 -07004545 zone->min_slab_pages = (realsize * sysctl_min_slab_ratio) / 100;
Christoph Lameter96146342006-07-03 00:24:13 -07004546#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07004547 zone->name = zone_names[j];
4548 spin_lock_init(&zone->lock);
4549 spin_lock_init(&zone->lru_lock);
Dave Hansenbdc8cb92005-10-29 18:16:53 -07004550 zone_seqlock_init(zone);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004551 zone->zone_pgdat = pgdat;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004552
Dave Hansened8ece22005-10-29 18:16:50 -07004553 zone_pcp_init(zone);
Hugh Dickins41113042012-01-12 17:20:01 -08004554 for_each_lru(lru)
4555 INIT_LIST_HEAD(&zone->lruvec.lists[lru]);
KOSAKI Motohiro6e901572009-01-07 18:08:15 -08004556 zone->reclaim_stat.recent_rotated[0] = 0;
4557 zone->reclaim_stat.recent_rotated[1] = 0;
4558 zone->reclaim_stat.recent_scanned[0] = 0;
4559 zone->reclaim_stat.recent_scanned[1] = 0;
Christoph Lameter2244b952006-06-30 01:55:33 -07004560 zap_zone_vm_stats(zone);
David Rientjese815af92007-10-16 23:25:54 -07004561 zone->flags = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004562 if (!size)
4563 continue;
4564
Mel Gormanba72cb82007-11-28 16:21:13 -08004565 set_pageblock_order(pageblock_default_order());
Mel Gorman835c1342007-10-16 01:25:47 -07004566 setup_usemap(pgdat, zone, size);
Dave Hansena2f3aa02007-01-10 23:15:30 -08004567 ret = init_currently_empty_zone(zone, zone_start_pfn,
4568 size, MEMMAP_EARLY);
Yasunori Goto718127c2006-06-23 02:03:10 -07004569 BUG_ON(ret);
Heiko Carstens76cdd582008-05-14 16:05:52 -07004570 memmap_init(size, nid, j, zone_start_pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004571 zone_start_pfn += size;
Linus Torvalds1da177e2005-04-16 15:20:36 -07004572 }
4573}
4574
Sam Ravnborg577a32f2007-05-17 23:29:25 +02004575static void __init_refok alloc_node_mem_map(struct pglist_data *pgdat)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004576{
Linus Torvalds1da177e2005-04-16 15:20:36 -07004577 /* Skip empty nodes */
4578 if (!pgdat->node_spanned_pages)
4579 return;
4580
Andy Whitcroftd41dee32005-06-23 00:07:54 -07004581#ifdef CONFIG_FLAT_NODE_MEM_MAP
Linus Torvalds1da177e2005-04-16 15:20:36 -07004582 /* ia64 gets its own node_mem_map, before this, without bootmem */
4583 if (!pgdat->node_mem_map) {
Bob Piccoe984bb42006-05-20 15:00:31 -07004584 unsigned long size, start, end;
Andy Whitcroftd41dee32005-06-23 00:07:54 -07004585 struct page *map;
4586
Bob Piccoe984bb42006-05-20 15:00:31 -07004587 /*
4588 * The zone's endpoints aren't required to be MAX_ORDER
4589 * aligned but the node_mem_map endpoints must be in order
4590 * for the buddy allocator to function correctly.
4591 */
4592 start = pgdat->node_start_pfn & ~(MAX_ORDER_NR_PAGES - 1);
4593 end = pgdat->node_start_pfn + pgdat->node_spanned_pages;
4594 end = ALIGN(end, MAX_ORDER_NR_PAGES);
4595 size = (end - start) * sizeof(struct page);
Dave Hansen6f167ec2005-06-23 00:07:39 -07004596 map = alloc_remap(pgdat->node_id, size);
4597 if (!map)
Yinghai Lu8f389a992011-05-11 15:13:32 -07004598 map = alloc_bootmem_node_nopanic(pgdat, size);
Bob Piccoe984bb42006-05-20 15:00:31 -07004599 pgdat->node_mem_map = map + (pgdat->node_start_pfn - start);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004600 }
Roman Zippel12d810c2007-05-31 00:40:54 -07004601#ifndef CONFIG_NEED_MULTIPLE_NODES
Linus Torvalds1da177e2005-04-16 15:20:36 -07004602 /*
4603 * With no DISCONTIG, the global mem_map is just set as node 0's
4604 */
Mel Gormanc7132162006-09-27 01:49:43 -07004605 if (pgdat == NODE_DATA(0)) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07004606 mem_map = NODE_DATA(0)->node_mem_map;
Tejun Heo0ee332c2011-12-08 10:22:09 -08004607#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
Mel Gormanc7132162006-09-27 01:49:43 -07004608 if (page_to_pfn(mem_map) != pgdat->node_start_pfn)
Thomas Bogendoerfer467bc462008-01-08 15:33:11 -08004609 mem_map -= (pgdat->node_start_pfn - ARCH_PFN_OFFSET);
Tejun Heo0ee332c2011-12-08 10:22:09 -08004610#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
Mel Gormanc7132162006-09-27 01:49:43 -07004611 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07004612#endif
Andy Whitcroftd41dee32005-06-23 00:07:54 -07004613#endif /* CONFIG_FLAT_NODE_MEM_MAP */
Linus Torvalds1da177e2005-04-16 15:20:36 -07004614}
4615
Johannes Weiner9109fb72008-07-23 21:27:20 -07004616void __paginginit free_area_init_node(int nid, unsigned long *zones_size,
4617 unsigned long node_start_pfn, unsigned long *zholes_size)
Linus Torvalds1da177e2005-04-16 15:20:36 -07004618{
Johannes Weiner9109fb72008-07-23 21:27:20 -07004619 pg_data_t *pgdat = NODE_DATA(nid);
4620
Linus Torvalds1da177e2005-04-16 15:20:36 -07004621 pgdat->node_id = nid;
4622 pgdat->node_start_pfn = node_start_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07004623 calculate_node_totalpages(pgdat, zones_size, zholes_size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07004624
4625 alloc_node_mem_map(pgdat);
Yinghai Lue8c27ac2008-06-01 13:15:22 -07004626#ifdef CONFIG_FLAT_NODE_MEM_MAP
4627 printk(KERN_DEBUG "free_area_init_node: node %d, pgdat %08lx, node_mem_map %08lx\n",
4628 nid, (unsigned long)pgdat,
4629 (unsigned long)pgdat->node_mem_map);
4630#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07004631
4632 free_area_init_core(pgdat, zones_size, zholes_size);
4633}
4634
Tejun Heo0ee332c2011-12-08 10:22:09 -08004635#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
Miklos Szeredi418508c2007-05-23 13:57:55 -07004636
4637#if MAX_NUMNODES > 1
4638/*
4639 * Figure out the number of possible node ids.
4640 */
4641static void __init setup_nr_node_ids(void)
4642{
4643 unsigned int node;
4644 unsigned int highest = 0;
4645
4646 for_each_node_mask(node, node_possible_map)
4647 highest = node;
4648 nr_node_ids = highest + 1;
4649}
4650#else
4651static inline void setup_nr_node_ids(void)
4652{
4653}
4654#endif
4655
Mel Gormanc7132162006-09-27 01:49:43 -07004656/**
Tejun Heo1e019792011-07-12 09:45:34 +02004657 * node_map_pfn_alignment - determine the maximum internode alignment
4658 *
4659 * This function should be called after node map is populated and sorted.
4660 * It calculates the maximum power of two alignment which can distinguish
4661 * all the nodes.
4662 *
4663 * For example, if all nodes are 1GiB and aligned to 1GiB, the return value
4664 * would indicate 1GiB alignment with (1 << (30 - PAGE_SHIFT)). If the
4665 * nodes are shifted by 256MiB, 256MiB. Note that if only the last node is
4666 * shifted, 1GiB is enough and this function will indicate so.
4667 *
4668 * This is used to test whether pfn -> nid mapping of the chosen memory
4669 * model has fine enough granularity to avoid incorrect mapping for the
4670 * populated node map.
4671 *
4672 * Returns the determined alignment in pfn's. 0 if there is no alignment
4673 * requirement (single node).
4674 */
4675unsigned long __init node_map_pfn_alignment(void)
4676{
4677 unsigned long accl_mask = 0, last_end = 0;
Tejun Heoc13291a2011-07-12 10:46:30 +02004678 unsigned long start, end, mask;
Tejun Heo1e019792011-07-12 09:45:34 +02004679 int last_nid = -1;
Tejun Heoc13291a2011-07-12 10:46:30 +02004680 int i, nid;
Tejun Heo1e019792011-07-12 09:45:34 +02004681
Tejun Heoc13291a2011-07-12 10:46:30 +02004682 for_each_mem_pfn_range(i, MAX_NUMNODES, &start, &end, &nid) {
Tejun Heo1e019792011-07-12 09:45:34 +02004683 if (!start || last_nid < 0 || last_nid == nid) {
4684 last_nid = nid;
4685 last_end = end;
4686 continue;
4687 }
4688
4689 /*
4690 * Start with a mask granular enough to pin-point to the
4691 * start pfn and tick off bits one-by-one until it becomes
4692 * too coarse to separate the current node from the last.
4693 */
4694 mask = ~((1 << __ffs(start)) - 1);
4695 while (mask && last_end <= (start & (mask << 1)))
4696 mask <<= 1;
4697
4698 /* accumulate all internode masks */
4699 accl_mask |= mask;
4700 }
4701
4702 /* convert mask to number of pages */
4703 return ~accl_mask + 1;
4704}
4705
Mel Gormana6af2bc2007-02-10 01:42:57 -08004706/* Find the lowest pfn for a node */
Adrian Bunkb69a7282008-07-23 21:28:12 -07004707static unsigned long __init find_min_pfn_for_node(int nid)
Mel Gormanc7132162006-09-27 01:49:43 -07004708{
Mel Gormana6af2bc2007-02-10 01:42:57 -08004709 unsigned long min_pfn = ULONG_MAX;
Tejun Heoc13291a2011-07-12 10:46:30 +02004710 unsigned long start_pfn;
4711 int i;
Mel Gorman1abbfb42006-11-23 12:01:41 +00004712
Tejun Heoc13291a2011-07-12 10:46:30 +02004713 for_each_mem_pfn_range(i, nid, &start_pfn, NULL, NULL)
4714 min_pfn = min(min_pfn, start_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07004715
Mel Gormana6af2bc2007-02-10 01:42:57 -08004716 if (min_pfn == ULONG_MAX) {
4717 printk(KERN_WARNING
Paul Jackson2bc0d262008-06-22 07:22:17 -07004718 "Could not find start_pfn for node %d\n", nid);
Mel Gormana6af2bc2007-02-10 01:42:57 -08004719 return 0;
4720 }
4721
4722 return min_pfn;
Mel Gormanc7132162006-09-27 01:49:43 -07004723}
4724
4725/**
4726 * find_min_pfn_with_active_regions - Find the minimum PFN registered
4727 *
4728 * It returns the minimum PFN based on information provided via
Randy Dunlap88ca3b92006-10-04 02:15:25 -07004729 * add_active_range().
Mel Gormanc7132162006-09-27 01:49:43 -07004730 */
4731unsigned long __init find_min_pfn_with_active_regions(void)
4732{
4733 return find_min_pfn_for_node(MAX_NUMNODES);
4734}
4735
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07004736/*
4737 * early_calculate_totalpages()
4738 * Sum pages in active regions for movable zone.
4739 * Populate N_HIGH_MEMORY for calculating usable_nodes.
4740 */
Adrian Bunk484f51f2007-10-16 01:26:03 -07004741static unsigned long __init early_calculate_totalpages(void)
Mel Gorman7e63efe2007-07-17 04:03:15 -07004742{
Mel Gorman7e63efe2007-07-17 04:03:15 -07004743 unsigned long totalpages = 0;
Tejun Heoc13291a2011-07-12 10:46:30 +02004744 unsigned long start_pfn, end_pfn;
4745 int i, nid;
Mel Gorman7e63efe2007-07-17 04:03:15 -07004746
Tejun Heoc13291a2011-07-12 10:46:30 +02004747 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) {
4748 unsigned long pages = end_pfn - start_pfn;
4749
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07004750 totalpages += pages;
4751 if (pages)
Tejun Heoc13291a2011-07-12 10:46:30 +02004752 node_set_state(nid, N_HIGH_MEMORY);
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07004753 }
4754 return totalpages;
Mel Gorman7e63efe2007-07-17 04:03:15 -07004755}
4756
Mel Gorman2a1e2742007-07-17 04:03:12 -07004757/*
4758 * Find the PFN the Movable zone begins in each node. Kernel memory
4759 * is spread evenly between nodes as long as the nodes have enough
4760 * memory. When they don't, some nodes will have more kernelcore than
4761 * others
4762 */
Kautuk Consulb224ef82012-03-21 16:34:15 -07004763static void __init find_zone_movable_pfns_for_nodes(void)
Mel Gorman2a1e2742007-07-17 04:03:12 -07004764{
4765 int i, nid;
4766 unsigned long usable_startpfn;
4767 unsigned long kernelcore_node, kernelcore_remaining;
Yinghai Lu66918dc2009-06-30 11:41:37 -07004768 /* save the state before borrow the nodemask */
4769 nodemask_t saved_node_state = node_states[N_HIGH_MEMORY];
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07004770 unsigned long totalpages = early_calculate_totalpages();
4771 int usable_nodes = nodes_weight(node_states[N_HIGH_MEMORY]);
Mel Gorman2a1e2742007-07-17 04:03:12 -07004772
Larry Bassele6436862011-10-14 10:59:07 -07004773#ifdef CONFIG_FIX_MOVABLE_ZONE
4774 required_movablecore = movable_reserved_size >> PAGE_SHIFT;
4775#endif
Mel Gorman7e63efe2007-07-17 04:03:15 -07004776 /*
4777 * If movablecore was specified, calculate what size of
4778 * kernelcore that corresponds so that memory usable for
4779 * any allocation type is evenly spread. If both kernelcore
4780 * and movablecore are specified, then the value of kernelcore
4781 * will be used for required_kernelcore if it's greater than
4782 * what movablecore would have allowed.
4783 */
4784 if (required_movablecore) {
Mel Gorman7e63efe2007-07-17 04:03:15 -07004785 unsigned long corepages;
4786
4787 /*
4788 * Round-up so that ZONE_MOVABLE is at least as large as what
4789 * was requested by the user
4790 */
4791 required_movablecore =
4792 roundup(required_movablecore, MAX_ORDER_NR_PAGES);
4793 corepages = totalpages - required_movablecore;
4794
4795 required_kernelcore = max(required_kernelcore, corepages);
4796 }
4797
Mel Gorman2a1e2742007-07-17 04:03:12 -07004798 /* If kernelcore was not specified, there is no ZONE_MOVABLE */
4799 if (!required_kernelcore)
Yinghai Lu66918dc2009-06-30 11:41:37 -07004800 goto out;
Mel Gorman2a1e2742007-07-17 04:03:12 -07004801
4802 /* usable_startpfn is the lowest possible pfn ZONE_MOVABLE can be at */
4803 find_usable_zone_for_movable();
4804 usable_startpfn = arch_zone_lowest_possible_pfn[movable_zone];
4805
4806restart:
4807 /* Spread kernelcore memory as evenly as possible throughout nodes */
4808 kernelcore_node = required_kernelcore / usable_nodes;
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07004809 for_each_node_state(nid, N_HIGH_MEMORY) {
Tejun Heoc13291a2011-07-12 10:46:30 +02004810 unsigned long start_pfn, end_pfn;
4811
Mel Gorman2a1e2742007-07-17 04:03:12 -07004812 /*
4813 * Recalculate kernelcore_node if the division per node
4814 * now exceeds what is necessary to satisfy the requested
4815 * amount of memory for the kernel
4816 */
4817 if (required_kernelcore < kernelcore_node)
4818 kernelcore_node = required_kernelcore / usable_nodes;
4819
4820 /*
4821 * As the map is walked, we track how much memory is usable
4822 * by the kernel using kernelcore_remaining. When it is
4823 * 0, the rest of the node is usable by ZONE_MOVABLE
4824 */
4825 kernelcore_remaining = kernelcore_node;
4826
4827 /* Go through each range of PFNs within this node */
Tejun Heoc13291a2011-07-12 10:46:30 +02004828 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
Mel Gorman2a1e2742007-07-17 04:03:12 -07004829 unsigned long size_pages;
4830
Tejun Heoc13291a2011-07-12 10:46:30 +02004831 start_pfn = max(start_pfn, zone_movable_pfn[nid]);
Mel Gorman2a1e2742007-07-17 04:03:12 -07004832 if (start_pfn >= end_pfn)
4833 continue;
4834
4835 /* Account for what is only usable for kernelcore */
4836 if (start_pfn < usable_startpfn) {
4837 unsigned long kernel_pages;
4838 kernel_pages = min(end_pfn, usable_startpfn)
4839 - start_pfn;
4840
4841 kernelcore_remaining -= min(kernel_pages,
4842 kernelcore_remaining);
4843 required_kernelcore -= min(kernel_pages,
4844 required_kernelcore);
4845
4846 /* Continue if range is now fully accounted */
4847 if (end_pfn <= usable_startpfn) {
4848
4849 /*
4850 * Push zone_movable_pfn to the end so
4851 * that if we have to rebalance
4852 * kernelcore across nodes, we will
4853 * not double account here
4854 */
4855 zone_movable_pfn[nid] = end_pfn;
4856 continue;
4857 }
4858 start_pfn = usable_startpfn;
4859 }
4860
4861 /*
4862 * The usable PFN range for ZONE_MOVABLE is from
4863 * start_pfn->end_pfn. Calculate size_pages as the
4864 * number of pages used as kernelcore
4865 */
4866 size_pages = end_pfn - start_pfn;
4867 if (size_pages > kernelcore_remaining)
4868 size_pages = kernelcore_remaining;
4869 zone_movable_pfn[nid] = start_pfn + size_pages;
4870
4871 /*
4872 * Some kernelcore has been met, update counts and
4873 * break if the kernelcore for this node has been
4874 * satisified
4875 */
4876 required_kernelcore -= min(required_kernelcore,
4877 size_pages);
4878 kernelcore_remaining -= size_pages;
4879 if (!kernelcore_remaining)
4880 break;
4881 }
4882 }
4883
4884 /*
4885 * If there is still required_kernelcore, we do another pass with one
4886 * less node in the count. This will push zone_movable_pfn[nid] further
4887 * along on the nodes that still have memory until kernelcore is
4888 * satisified
4889 */
4890 usable_nodes--;
4891 if (usable_nodes && required_kernelcore > usable_nodes)
4892 goto restart;
4893
4894 /* Align start of ZONE_MOVABLE on all nids to MAX_ORDER_NR_PAGES */
4895 for (nid = 0; nid < MAX_NUMNODES; nid++)
4896 zone_movable_pfn[nid] =
4897 roundup(zone_movable_pfn[nid], MAX_ORDER_NR_PAGES);
Yinghai Lu66918dc2009-06-30 11:41:37 -07004898
4899out:
4900 /* restore the node_state */
4901 node_states[N_HIGH_MEMORY] = saved_node_state;
Mel Gorman2a1e2742007-07-17 04:03:12 -07004902}
4903
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07004904/* Any regular memory on that node ? */
4905static void check_for_regular_memory(pg_data_t *pgdat)
4906{
4907#ifdef CONFIG_HIGHMEM
4908 enum zone_type zone_type;
4909
4910 for (zone_type = 0; zone_type <= ZONE_NORMAL; zone_type++) {
4911 struct zone *zone = &pgdat->node_zones[zone_type];
Bob Liud0048b02012-01-12 17:19:07 -08004912 if (zone->present_pages) {
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07004913 node_set_state(zone_to_nid(zone), N_NORMAL_MEMORY);
Bob Liud0048b02012-01-12 17:19:07 -08004914 break;
4915 }
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07004916 }
4917#endif
4918}
4919
Mel Gormanc7132162006-09-27 01:49:43 -07004920/**
4921 * free_area_init_nodes - Initialise all pg_data_t and zone data
Randy Dunlap88ca3b92006-10-04 02:15:25 -07004922 * @max_zone_pfn: an array of max PFNs for each zone
Mel Gormanc7132162006-09-27 01:49:43 -07004923 *
4924 * This will call free_area_init_node() for each active node in the system.
4925 * Using the page ranges provided by add_active_range(), the size of each
4926 * zone in each node and their holes is calculated. If the maximum PFN
4927 * between two adjacent zones match, it is assumed that the zone is empty.
4928 * For example, if arch_max_dma_pfn == arch_max_dma32_pfn, it is assumed
4929 * that arch_max_dma32_pfn has no pages. It is also assumed that a zone
4930 * starts where the previous one ended. For example, ZONE_DMA32 starts
4931 * at arch_max_dma_pfn.
4932 */
4933void __init free_area_init_nodes(unsigned long *max_zone_pfn)
4934{
Tejun Heoc13291a2011-07-12 10:46:30 +02004935 unsigned long start_pfn, end_pfn;
4936 int i, nid;
Mel Gormana6af2bc2007-02-10 01:42:57 -08004937
Mel Gormanc7132162006-09-27 01:49:43 -07004938 /* Record where the zone boundaries are */
4939 memset(arch_zone_lowest_possible_pfn, 0,
4940 sizeof(arch_zone_lowest_possible_pfn));
4941 memset(arch_zone_highest_possible_pfn, 0,
4942 sizeof(arch_zone_highest_possible_pfn));
4943 arch_zone_lowest_possible_pfn[0] = find_min_pfn_with_active_regions();
4944 arch_zone_highest_possible_pfn[0] = max_zone_pfn[0];
4945 for (i = 1; i < MAX_NR_ZONES; i++) {
Mel Gorman2a1e2742007-07-17 04:03:12 -07004946 if (i == ZONE_MOVABLE)
4947 continue;
Mel Gormanc7132162006-09-27 01:49:43 -07004948 arch_zone_lowest_possible_pfn[i] =
4949 arch_zone_highest_possible_pfn[i-1];
4950 arch_zone_highest_possible_pfn[i] =
4951 max(max_zone_pfn[i], arch_zone_lowest_possible_pfn[i]);
4952 }
Mel Gorman2a1e2742007-07-17 04:03:12 -07004953 arch_zone_lowest_possible_pfn[ZONE_MOVABLE] = 0;
4954 arch_zone_highest_possible_pfn[ZONE_MOVABLE] = 0;
4955
4956 /* Find the PFNs that ZONE_MOVABLE begins at in each node */
4957 memset(zone_movable_pfn, 0, sizeof(zone_movable_pfn));
Kautuk Consulb224ef82012-03-21 16:34:15 -07004958 find_zone_movable_pfns_for_nodes();
Mel Gormanc7132162006-09-27 01:49:43 -07004959
Mel Gormanc7132162006-09-27 01:49:43 -07004960 /* Print out the zone ranges */
4961 printk("Zone PFN ranges:\n");
Mel Gorman2a1e2742007-07-17 04:03:12 -07004962 for (i = 0; i < MAX_NR_ZONES; i++) {
4963 if (i == ZONE_MOVABLE)
4964 continue;
David Rientjes72f0ba02010-03-05 13:42:14 -08004965 printk(" %-8s ", zone_names[i]);
4966 if (arch_zone_lowest_possible_pfn[i] ==
4967 arch_zone_highest_possible_pfn[i])
4968 printk("empty\n");
4969 else
4970 printk("%0#10lx -> %0#10lx\n",
Mel Gormanc7132162006-09-27 01:49:43 -07004971 arch_zone_lowest_possible_pfn[i],
4972 arch_zone_highest_possible_pfn[i]);
Mel Gorman2a1e2742007-07-17 04:03:12 -07004973 }
4974
4975 /* Print out the PFNs ZONE_MOVABLE begins at in each node */
4976 printk("Movable zone start PFN for each node\n");
4977 for (i = 0; i < MAX_NUMNODES; i++) {
4978 if (zone_movable_pfn[i])
4979 printk(" Node %d: %lu\n", i, zone_movable_pfn[i]);
4980 }
Mel Gormanc7132162006-09-27 01:49:43 -07004981
4982 /* Print out the early_node_map[] */
Tejun Heoc13291a2011-07-12 10:46:30 +02004983 printk("Early memory PFN ranges\n");
4984 for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid)
4985 printk(" %3d: %0#10lx -> %0#10lx\n", nid, start_pfn, end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07004986
4987 /* Initialise every node */
Mel Gorman708614e2008-07-23 21:26:51 -07004988 mminit_verify_pageflags_layout();
Christoph Lameter8ef82862007-02-20 13:57:52 -08004989 setup_nr_node_ids();
Mel Gormanc7132162006-09-27 01:49:43 -07004990 for_each_online_node(nid) {
4991 pg_data_t *pgdat = NODE_DATA(nid);
Johannes Weiner9109fb72008-07-23 21:27:20 -07004992 free_area_init_node(nid, NULL,
Mel Gormanc7132162006-09-27 01:49:43 -07004993 find_min_pfn_for_node(nid), NULL);
Lee Schermerhorn37b07e42007-10-16 01:25:39 -07004994
4995 /* Any memory on that node */
4996 if (pgdat->node_present_pages)
4997 node_set_state(nid, N_HIGH_MEMORY);
4998 check_for_regular_memory(pgdat);
Mel Gormanc7132162006-09-27 01:49:43 -07004999 }
5000}
Mel Gorman2a1e2742007-07-17 04:03:12 -07005001
Mel Gorman7e63efe2007-07-17 04:03:15 -07005002static int __init cmdline_parse_core(char *p, unsigned long *core)
Mel Gorman2a1e2742007-07-17 04:03:12 -07005003{
5004 unsigned long long coremem;
5005 if (!p)
5006 return -EINVAL;
5007
5008 coremem = memparse(p, &p);
Mel Gorman7e63efe2007-07-17 04:03:15 -07005009 *core = coremem >> PAGE_SHIFT;
Mel Gorman2a1e2742007-07-17 04:03:12 -07005010
Mel Gorman7e63efe2007-07-17 04:03:15 -07005011 /* Paranoid check that UL is enough for the coremem value */
Mel Gorman2a1e2742007-07-17 04:03:12 -07005012 WARN_ON((coremem >> PAGE_SHIFT) > ULONG_MAX);
5013
5014 return 0;
5015}
Mel Gormaned7ed362007-07-17 04:03:14 -07005016
Mel Gorman7e63efe2007-07-17 04:03:15 -07005017/*
5018 * kernelcore=size sets the amount of memory for use for allocations that
5019 * cannot be reclaimed or migrated.
5020 */
5021static int __init cmdline_parse_kernelcore(char *p)
5022{
5023 return cmdline_parse_core(p, &required_kernelcore);
5024}
5025
5026/*
5027 * movablecore=size sets the amount of memory for use for allocations that
5028 * can be reclaimed or migrated.
5029 */
5030static int __init cmdline_parse_movablecore(char *p)
5031{
5032 return cmdline_parse_core(p, &required_movablecore);
5033}
5034
Mel Gormaned7ed362007-07-17 04:03:14 -07005035early_param("kernelcore", cmdline_parse_kernelcore);
Mel Gorman7e63efe2007-07-17 04:03:15 -07005036early_param("movablecore", cmdline_parse_movablecore);
Mel Gormaned7ed362007-07-17 04:03:14 -07005037
Tejun Heo0ee332c2011-12-08 10:22:09 -08005038#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
Mel Gormanc7132162006-09-27 01:49:43 -07005039
Mel Gorman0e0b8642006-09-27 01:49:56 -07005040/**
Randy Dunlap88ca3b92006-10-04 02:15:25 -07005041 * set_dma_reserve - set the specified number of pages reserved in the first zone
5042 * @new_dma_reserve: The number of pages to mark reserved
Mel Gorman0e0b8642006-09-27 01:49:56 -07005043 *
5044 * The per-cpu batchsize and zone watermarks are determined by present_pages.
5045 * In the DMA zone, a significant percentage may be consumed by kernel image
5046 * and other unfreeable allocations which can skew the watermarks badly. This
Randy Dunlap88ca3b92006-10-04 02:15:25 -07005047 * function may optionally be used to account for unfreeable pages in the
5048 * first zone (e.g., ZONE_DMA). The effect will be lower watermarks and
5049 * smaller per-cpu batchsize.
Mel Gorman0e0b8642006-09-27 01:49:56 -07005050 */
5051void __init set_dma_reserve(unsigned long new_dma_reserve)
5052{
5053 dma_reserve = new_dma_reserve;
5054}
5055
Linus Torvalds1da177e2005-04-16 15:20:36 -07005056void __init free_area_init(unsigned long *zones_size)
5057{
Johannes Weiner9109fb72008-07-23 21:27:20 -07005058 free_area_init_node(0, zones_size,
Linus Torvalds1da177e2005-04-16 15:20:36 -07005059 __pa(PAGE_OFFSET) >> PAGE_SHIFT, NULL);
5060}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005061
Linus Torvalds1da177e2005-04-16 15:20:36 -07005062static int page_alloc_cpu_notify(struct notifier_block *self,
5063 unsigned long action, void *hcpu)
5064{
5065 int cpu = (unsigned long)hcpu;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005066
Rafael J. Wysocki8bb78442007-05-09 02:35:10 -07005067 if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
Konstantin Khlebnikovf0cb3c72012-03-21 16:34:06 -07005068 lru_add_drain_cpu(cpu);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08005069 drain_pages(cpu);
5070
5071 /*
5072 * Spill the event counters of the dead processor
5073 * into the current processors event counters.
5074 * This artificially elevates the count of the current
5075 * processor.
5076 */
Christoph Lameterf8891e52006-06-30 01:55:45 -07005077 vm_events_fold_cpu(cpu);
Christoph Lameter9f8f2172008-02-04 22:29:11 -08005078
5079 /*
5080 * Zero the differential counters of the dead processor
5081 * so that the vm statistics are consistent.
5082 *
5083 * This is only okay since the processor is dead and cannot
5084 * race with what we are doing.
5085 */
Christoph Lameter2244b952006-06-30 01:55:33 -07005086 refresh_cpu_vm_stats(cpu);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005087 }
5088 return NOTIFY_OK;
5089}
Linus Torvalds1da177e2005-04-16 15:20:36 -07005090
5091void __init page_alloc_init(void)
5092{
5093 hotcpu_notifier(page_alloc_cpu_notify, 0);
5094}
5095
5096/*
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07005097 * calculate_totalreserve_pages - called when sysctl_lower_zone_reserve_ratio
5098 * or min_free_kbytes changes.
5099 */
5100static void calculate_totalreserve_pages(void)
5101{
5102 struct pglist_data *pgdat;
5103 unsigned long reserve_pages = 0;
Christoph Lameter2f6726e2006-09-25 23:31:18 -07005104 enum zone_type i, j;
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07005105
5106 for_each_online_pgdat(pgdat) {
5107 for (i = 0; i < MAX_NR_ZONES; i++) {
5108 struct zone *zone = pgdat->node_zones + i;
5109 unsigned long max = 0;
5110
5111 /* Find valid and maximum lowmem_reserve in the zone */
5112 for (j = i; j < MAX_NR_ZONES; j++) {
5113 if (zone->lowmem_reserve[j] > max)
5114 max = zone->lowmem_reserve[j];
5115 }
5116
Mel Gorman41858962009-06-16 15:32:12 -07005117 /* we treat the high watermark as reserved pages. */
5118 max += high_wmark_pages(zone);
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07005119
5120 if (max > zone->present_pages)
5121 max = zone->present_pages;
5122 reserve_pages += max;
Johannes Weinerab8fabd2012-01-10 15:07:42 -08005123 /*
5124 * Lowmem reserves are not available to
5125 * GFP_HIGHUSER page cache allocations and
5126 * kswapd tries to balance zones to their high
5127 * watermark. As a result, neither should be
5128 * regarded as dirtyable memory, to prevent a
5129 * situation where reclaim has to clean pages
5130 * in order to balance the zones.
5131 */
5132 zone->dirty_balance_reserve = max;
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07005133 }
5134 }
Johannes Weinerab8fabd2012-01-10 15:07:42 -08005135 dirty_balance_reserve = reserve_pages;
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07005136 totalreserve_pages = reserve_pages;
5137}
5138
5139/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07005140 * setup_per_zone_lowmem_reserve - called whenever
5141 * sysctl_lower_zone_reserve_ratio changes. Ensures that each zone
5142 * has a correct pages reserved value, so an adequate number of
5143 * pages are left in the zone after a successful __alloc_pages().
5144 */
5145static void setup_per_zone_lowmem_reserve(void)
5146{
5147 struct pglist_data *pgdat;
Christoph Lameter2f6726e2006-09-25 23:31:18 -07005148 enum zone_type j, idx;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005149
KAMEZAWA Hiroyukiec936fc2006-03-27 01:15:59 -08005150 for_each_online_pgdat(pgdat) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005151 for (j = 0; j < MAX_NR_ZONES; j++) {
5152 struct zone *zone = pgdat->node_zones + j;
5153 unsigned long present_pages = zone->present_pages;
5154
5155 zone->lowmem_reserve[j] = 0;
5156
Christoph Lameter2f6726e2006-09-25 23:31:18 -07005157 idx = j;
5158 while (idx) {
Linus Torvalds1da177e2005-04-16 15:20:36 -07005159 struct zone *lower_zone;
5160
Christoph Lameter2f6726e2006-09-25 23:31:18 -07005161 idx--;
5162
Linus Torvalds1da177e2005-04-16 15:20:36 -07005163 if (sysctl_lowmem_reserve_ratio[idx] < 1)
5164 sysctl_lowmem_reserve_ratio[idx] = 1;
5165
5166 lower_zone = pgdat->node_zones + idx;
5167 lower_zone->lowmem_reserve[j] = present_pages /
5168 sysctl_lowmem_reserve_ratio[idx];
5169 present_pages += lower_zone->present_pages;
5170 }
5171 }
5172 }
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07005173
5174 /* update totalreserve_pages */
5175 calculate_totalreserve_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005176}
5177
Mel Gormane12aade2011-04-25 21:36:42 +00005178static void __setup_per_zone_wmarks(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005179{
5180 unsigned long pages_min = min_free_kbytes >> (PAGE_SHIFT - 10);
5181 unsigned long lowmem_pages = 0;
5182 struct zone *zone;
5183 unsigned long flags;
5184
5185 /* Calculate total number of !ZONE_HIGHMEM pages */
5186 for_each_zone(zone) {
5187 if (!is_highmem(zone))
5188 lowmem_pages += zone->present_pages;
5189 }
5190
5191 for_each_zone(zone) {
Andrew Mortonac924c62006-05-15 09:43:59 -07005192 u64 tmp;
5193
Gerald Schaefer1125b4e2008-10-18 20:27:11 -07005194 spin_lock_irqsave(&zone->lock, flags);
Andrew Mortonac924c62006-05-15 09:43:59 -07005195 tmp = (u64)pages_min * zone->present_pages;
5196 do_div(tmp, lowmem_pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005197 if (is_highmem(zone)) {
5198 /*
Nick Piggin669ed172005-11-13 16:06:45 -08005199 * __GFP_HIGH and PF_MEMALLOC allocations usually don't
5200 * need highmem pages, so cap pages_min to a small
5201 * value here.
5202 *
Mel Gorman41858962009-06-16 15:32:12 -07005203 * The WMARK_HIGH-WMARK_LOW and (WMARK_LOW-WMARK_MIN)
Nick Piggin669ed172005-11-13 16:06:45 -08005204 * deltas controls asynch page reclaim, and so should
5205 * not be capped for highmem.
Linus Torvalds1da177e2005-04-16 15:20:36 -07005206 */
5207 int min_pages;
5208
5209 min_pages = zone->present_pages / 1024;
5210 if (min_pages < SWAP_CLUSTER_MAX)
5211 min_pages = SWAP_CLUSTER_MAX;
5212 if (min_pages > 128)
5213 min_pages = 128;
Mel Gorman41858962009-06-16 15:32:12 -07005214 zone->watermark[WMARK_MIN] = min_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005215 } else {
Nick Piggin669ed172005-11-13 16:06:45 -08005216 /*
5217 * If it's a lowmem zone, reserve a number of pages
Linus Torvalds1da177e2005-04-16 15:20:36 -07005218 * proportionate to the zone's size.
5219 */
Mel Gorman41858962009-06-16 15:32:12 -07005220 zone->watermark[WMARK_MIN] = tmp;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005221 }
5222
Mel Gorman41858962009-06-16 15:32:12 -07005223 zone->watermark[WMARK_LOW] = min_wmark_pages(zone) + (tmp >> 2);
5224 zone->watermark[WMARK_HIGH] = min_wmark_pages(zone) + (tmp >> 1);
Marek Szyprowskif1f63882012-01-25 12:49:24 +01005225
Mel Gorman56fd56b2007-10-16 01:25:58 -07005226 setup_zone_migrate_reserve(zone);
Gerald Schaefer1125b4e2008-10-18 20:27:11 -07005227 spin_unlock_irqrestore(&zone->lock, flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005228 }
Hideo AOKIcb45b0e2006-04-10 22:52:59 -07005229
5230 /* update totalreserve_pages */
5231 calculate_totalreserve_pages();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005232}
5233
Mel Gormane12aade2011-04-25 21:36:42 +00005234/**
5235 * setup_per_zone_wmarks - called when min_free_kbytes changes
5236 * or when memory is hot-{added|removed}
5237 *
5238 * Ensures that the watermark[min,low,high] values for each zone are set
5239 * correctly with respect to min_free_kbytes.
5240 */
5241void setup_per_zone_wmarks(void)
5242{
5243 mutex_lock(&zonelists_mutex);
5244 __setup_per_zone_wmarks();
5245 mutex_unlock(&zonelists_mutex);
5246}
5247
Randy Dunlap55a44622009-09-21 17:01:20 -07005248/*
Rik van Riel556adec2008-10-18 20:26:34 -07005249 * The inactive anon list should be small enough that the VM never has to
5250 * do too much work, but large enough that each inactive page has a chance
5251 * to be referenced again before it is swapped out.
5252 *
5253 * The inactive_anon ratio is the target ratio of ACTIVE_ANON to
5254 * INACTIVE_ANON pages on this zone's LRU, maintained by the
5255 * pageout code. A zone->inactive_ratio of 3 means 3:1 or 25% of
5256 * the anonymous pages are kept on the inactive list.
5257 *
5258 * total target max
5259 * memory ratio inactive anon
5260 * -------------------------------------
5261 * 10MB 1 5MB
5262 * 100MB 1 50MB
5263 * 1GB 3 250MB
5264 * 10GB 10 0.9GB
5265 * 100GB 31 3GB
5266 * 1TB 101 10GB
5267 * 10TB 320 32GB
5268 */
KOSAKI Motohiro1b79acc2011-05-24 17:11:32 -07005269static void __meminit calculate_zone_inactive_ratio(struct zone *zone)
Minchan Kim96cb4df2009-06-16 15:32:49 -07005270{
5271 unsigned int gb, ratio;
5272
5273 /* Zone size in gigabytes */
5274 gb = zone->present_pages >> (30 - PAGE_SHIFT);
5275 if (gb)
5276 ratio = int_sqrt(10 * gb);
5277 else
5278 ratio = 1;
5279
5280 zone->inactive_ratio = ratio;
5281}
5282
KOSAKI Motohiro839a4fc2011-05-24 17:11:31 -07005283static void __meminit setup_per_zone_inactive_ratio(void)
Rik van Riel556adec2008-10-18 20:26:34 -07005284{
5285 struct zone *zone;
5286
Minchan Kim96cb4df2009-06-16 15:32:49 -07005287 for_each_zone(zone)
5288 calculate_zone_inactive_ratio(zone);
Rik van Riel556adec2008-10-18 20:26:34 -07005289}
5290
Linus Torvalds1da177e2005-04-16 15:20:36 -07005291/*
5292 * Initialise min_free_kbytes.
5293 *
5294 * For small machines we want it small (128k min). For large machines
5295 * we want it large (64MB max). But it is not linear, because network
5296 * bandwidth does not increase linearly with machine size. We use
5297 *
5298 * min_free_kbytes = 4 * sqrt(lowmem_kbytes), for better accuracy:
5299 * min_free_kbytes = sqrt(lowmem_kbytes * 16)
5300 *
5301 * which yields
5302 *
5303 * 16MB: 512k
5304 * 32MB: 724k
5305 * 64MB: 1024k
5306 * 128MB: 1448k
5307 * 256MB: 2048k
5308 * 512MB: 2896k
5309 * 1024MB: 4096k
5310 * 2048MB: 5792k
5311 * 4096MB: 8192k
5312 * 8192MB: 11584k
5313 * 16384MB: 16384k
5314 */
KOSAKI Motohiro1b79acc2011-05-24 17:11:32 -07005315int __meminit init_per_zone_wmark_min(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005316{
5317 unsigned long lowmem_kbytes;
5318
5319 lowmem_kbytes = nr_free_buffer_pages() * (PAGE_SIZE >> 10);
5320
5321 min_free_kbytes = int_sqrt(lowmem_kbytes * 16);
5322 if (min_free_kbytes < 128)
5323 min_free_kbytes = 128;
5324 if (min_free_kbytes > 65536)
5325 min_free_kbytes = 65536;
Minchan Kimbc75d332009-06-16 15:32:48 -07005326 setup_per_zone_wmarks();
KOSAKI Motohiroa6cccdc2011-05-24 17:11:33 -07005327 refresh_zone_stat_thresholds();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005328 setup_per_zone_lowmem_reserve();
Rik van Riel556adec2008-10-18 20:26:34 -07005329 setup_per_zone_inactive_ratio();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005330 return 0;
5331}
Minchan Kimbc75d332009-06-16 15:32:48 -07005332module_init(init_per_zone_wmark_min)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005333
5334/*
5335 * min_free_kbytes_sysctl_handler - just a wrapper around proc_dointvec() so
5336 * that we can call two helper functions whenever min_free_kbytes
5337 * changes.
5338 */
5339int min_free_kbytes_sysctl_handler(ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07005340 void __user *buffer, size_t *length, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005341{
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07005342 proc_dointvec(table, write, buffer, length, ppos);
Mel Gorman3b1d92c2007-05-06 14:49:30 -07005343 if (write)
Minchan Kimbc75d332009-06-16 15:32:48 -07005344 setup_per_zone_wmarks();
Linus Torvalds1da177e2005-04-16 15:20:36 -07005345 return 0;
5346}
5347
Christoph Lameter96146342006-07-03 00:24:13 -07005348#ifdef CONFIG_NUMA
5349int sysctl_min_unmapped_ratio_sysctl_handler(ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07005350 void __user *buffer, size_t *length, loff_t *ppos)
Christoph Lameter96146342006-07-03 00:24:13 -07005351{
5352 struct zone *zone;
5353 int rc;
5354
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07005355 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
Christoph Lameter96146342006-07-03 00:24:13 -07005356 if (rc)
5357 return rc;
5358
5359 for_each_zone(zone)
Christoph Lameter8417bba2006-09-25 23:31:51 -07005360 zone->min_unmapped_pages = (zone->present_pages *
Christoph Lameter96146342006-07-03 00:24:13 -07005361 sysctl_min_unmapped_ratio) / 100;
5362 return 0;
5363}
Christoph Lameter0ff38492006-09-25 23:31:52 -07005364
5365int sysctl_min_slab_ratio_sysctl_handler(ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07005366 void __user *buffer, size_t *length, loff_t *ppos)
Christoph Lameter0ff38492006-09-25 23:31:52 -07005367{
5368 struct zone *zone;
5369 int rc;
5370
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07005371 rc = proc_dointvec_minmax(table, write, buffer, length, ppos);
Christoph Lameter0ff38492006-09-25 23:31:52 -07005372 if (rc)
5373 return rc;
5374
5375 for_each_zone(zone)
5376 zone->min_slab_pages = (zone->present_pages *
5377 sysctl_min_slab_ratio) / 100;
5378 return 0;
5379}
Christoph Lameter96146342006-07-03 00:24:13 -07005380#endif
5381
Linus Torvalds1da177e2005-04-16 15:20:36 -07005382/*
5383 * lowmem_reserve_ratio_sysctl_handler - just a wrapper around
5384 * proc_dointvec() so that we can call setup_per_zone_lowmem_reserve()
5385 * whenever sysctl_lowmem_reserve_ratio changes.
5386 *
5387 * The reserve ratio obviously has absolutely no relation with the
Mel Gorman41858962009-06-16 15:32:12 -07005388 * minimum watermarks. The lowmem reserve ratio can only make sense
Linus Torvalds1da177e2005-04-16 15:20:36 -07005389 * if in function of the boot time zone sizes.
5390 */
5391int lowmem_reserve_ratio_sysctl_handler(ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07005392 void __user *buffer, size_t *length, loff_t *ppos)
Linus Torvalds1da177e2005-04-16 15:20:36 -07005393{
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07005394 proc_dointvec_minmax(table, write, buffer, length, ppos);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005395 setup_per_zone_lowmem_reserve();
5396 return 0;
5397}
5398
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08005399/*
5400 * percpu_pagelist_fraction - changes the pcp->high for each zone on each
5401 * cpu. It is the fraction of total pages in each zone that a hot per cpu pagelist
5402 * can have before it gets flushed back to buddy allocator.
5403 */
5404
5405int percpu_pagelist_fraction_sysctl_handler(ctl_table *table, int write,
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07005406 void __user *buffer, size_t *length, loff_t *ppos)
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08005407{
5408 struct zone *zone;
5409 unsigned int cpu;
5410 int ret;
5411
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07005412 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
Sasha Levin93278812012-05-10 13:01:44 -07005413 if (!write || (ret < 0))
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08005414 return ret;
Dimitri Sivanich364df0e2009-06-23 12:37:04 -07005415 for_each_populated_zone(zone) {
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005416 for_each_possible_cpu(cpu) {
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08005417 unsigned long high;
5418 high = zone->present_pages / percpu_pagelist_fraction;
Christoph Lameter99dcc3e2010-01-05 15:34:51 +09005419 setup_pagelist_highmark(
5420 per_cpu_ptr(zone->pageset, cpu), high);
Rohit Seth8ad4b1f2006-01-08 01:00:40 -08005421 }
5422 }
5423 return 0;
5424}
5425
David S. Millerf034b5d2006-08-24 03:08:07 -07005426int hashdist = HASHDIST_DEFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005427
5428#ifdef CONFIG_NUMA
5429static int __init set_hashdist(char *str)
5430{
5431 if (!str)
5432 return 0;
5433 hashdist = simple_strtoul(str, &str, 0);
5434 return 1;
5435}
5436__setup("hashdist=", set_hashdist);
5437#endif
5438
5439/*
5440 * allocate a large system hash table from bootmem
5441 * - it is assumed that the hash table must contain an exact power-of-2
5442 * quantity of entries
5443 * - limit is the number of hash buckets, not the total allocation size
5444 */
5445void *__init alloc_large_system_hash(const char *tablename,
5446 unsigned long bucketsize,
5447 unsigned long numentries,
5448 int scale,
5449 int flags,
5450 unsigned int *_hash_shift,
5451 unsigned int *_hash_mask,
5452 unsigned long limit)
5453{
5454 unsigned long long max = limit;
5455 unsigned long log2qty, size;
5456 void *table = NULL;
5457
5458 /* allow the kernel cmdline to have a say */
5459 if (!numentries) {
5460 /* round applicable memory size up to nearest megabyte */
Andrew Morton04903662006-12-06 20:37:33 -08005461 numentries = nr_kernel_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005462 numentries += (1UL << (20 - PAGE_SHIFT)) - 1;
5463 numentries >>= 20 - PAGE_SHIFT;
5464 numentries <<= 20 - PAGE_SHIFT;
5465
5466 /* limit to 1 bucket per 2^scale bytes of low memory */
5467 if (scale > PAGE_SHIFT)
5468 numentries >>= (scale - PAGE_SHIFT);
5469 else
5470 numentries <<= (PAGE_SHIFT - scale);
Paul Mundt9ab37b82007-01-05 16:36:30 -08005471
5472 /* Make sure we've got at least a 0-order allocation.. */
Jan Beulich2c85f512009-09-21 17:03:07 -07005473 if (unlikely(flags & HASH_SMALL)) {
5474 /* Makes no sense without HASH_EARLY */
5475 WARN_ON(!(flags & HASH_EARLY));
5476 if (!(numentries >> *_hash_shift)) {
5477 numentries = 1UL << *_hash_shift;
5478 BUG_ON(!numentries);
5479 }
5480 } else if (unlikely((numentries * bucketsize) < PAGE_SIZE))
Paul Mundt9ab37b82007-01-05 16:36:30 -08005481 numentries = PAGE_SIZE / bucketsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -07005482 }
John Hawkes6e692ed2006-03-25 03:08:02 -08005483 numentries = roundup_pow_of_two(numentries);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005484
5485 /* limit allocation size to 1/16 total memory by default */
5486 if (max == 0) {
5487 max = ((unsigned long long)nr_all_pages << PAGE_SHIFT) >> 4;
5488 do_div(max, bucketsize);
5489 }
Dimitri Sivanich074b8512012-02-08 12:39:07 -08005490 max = min(max, 0x80000000ULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005491
5492 if (numentries > max)
5493 numentries = max;
5494
David Howellsf0d1b0b2006-12-08 02:37:49 -08005495 log2qty = ilog2(numentries);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005496
5497 do {
5498 size = bucketsize << log2qty;
5499 if (flags & HASH_EARLY)
Jan Beulich74768ed2008-08-12 15:08:39 -07005500 table = alloc_bootmem_nopanic(size);
Linus Torvalds1da177e2005-04-16 15:20:36 -07005501 else if (hashdist)
5502 table = __vmalloc(size, GFP_ATOMIC, PAGE_KERNEL);
5503 else {
Eric Dumazet1037b832007-07-15 23:38:05 -07005504 /*
5505 * If bucketsize is not a power-of-two, we may free
Mel Gormana1dd2682009-06-16 15:32:19 -07005506 * some pages at the end of hash table which
5507 * alloc_pages_exact() automatically does
Eric Dumazet1037b832007-07-15 23:38:05 -07005508 */
Catalin Marinas264ef8a2009-07-07 10:33:01 +01005509 if (get_order(size) < MAX_ORDER) {
Mel Gormana1dd2682009-06-16 15:32:19 -07005510 table = alloc_pages_exact(size, GFP_ATOMIC);
Catalin Marinas264ef8a2009-07-07 10:33:01 +01005511 kmemleak_alloc(table, size, 1, GFP_ATOMIC);
5512 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07005513 }
5514 } while (!table && size > PAGE_SIZE && --log2qty);
5515
5516 if (!table)
5517 panic("Failed to allocate %s hash table\n", tablename);
5518
Robin Holtf241e662010-10-07 12:59:26 -07005519 printk(KERN_INFO "%s hash table entries: %ld (order: %d, %lu bytes)\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -07005520 tablename,
Robin Holtf241e662010-10-07 12:59:26 -07005521 (1UL << log2qty),
David Howellsf0d1b0b2006-12-08 02:37:49 -08005522 ilog2(size) - PAGE_SHIFT,
Linus Torvalds1da177e2005-04-16 15:20:36 -07005523 size);
5524
5525 if (_hash_shift)
5526 *_hash_shift = log2qty;
5527 if (_hash_mask)
5528 *_hash_mask = (1 << log2qty) - 1;
5529
5530 return table;
5531}
KAMEZAWA Hiroyukia117e662006-03-27 01:15:25 -08005532
Mel Gorman835c1342007-10-16 01:25:47 -07005533/* Return a pointer to the bitmap storing bits affecting a block of pages */
5534static inline unsigned long *get_pageblock_bitmap(struct zone *zone,
5535 unsigned long pfn)
5536{
5537#ifdef CONFIG_SPARSEMEM
5538 return __pfn_to_section(pfn)->pageblock_flags;
5539#else
5540 return zone->pageblock_flags;
5541#endif /* CONFIG_SPARSEMEM */
5542}
Andrew Morton6220ec72006-10-19 23:29:05 -07005543
Mel Gorman835c1342007-10-16 01:25:47 -07005544static inline int pfn_to_bitidx(struct zone *zone, unsigned long pfn)
5545{
5546#ifdef CONFIG_SPARSEMEM
5547 pfn &= (PAGES_PER_SECTION-1);
Mel Gormand9c23402007-10-16 01:26:01 -07005548 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
Mel Gorman835c1342007-10-16 01:25:47 -07005549#else
Laura Abbott5bca7222012-11-30 14:07:01 -08005550 pfn = pfn - round_down(zone->zone_start_pfn, pageblock_nr_pages);
Mel Gormand9c23402007-10-16 01:26:01 -07005551 return (pfn >> pageblock_order) * NR_PAGEBLOCK_BITS;
Mel Gorman835c1342007-10-16 01:25:47 -07005552#endif /* CONFIG_SPARSEMEM */
5553}
5554
5555/**
Mel Gormand9c23402007-10-16 01:26:01 -07005556 * get_pageblock_flags_group - Return the requested group of flags for the pageblock_nr_pages block of pages
Mel Gorman835c1342007-10-16 01:25:47 -07005557 * @page: The page within the block of interest
5558 * @start_bitidx: The first bit of interest to retrieve
5559 * @end_bitidx: The last bit of interest
5560 * returns pageblock_bits flags
5561 */
5562unsigned long get_pageblock_flags_group(struct page *page,
5563 int start_bitidx, int end_bitidx)
5564{
5565 struct zone *zone;
5566 unsigned long *bitmap;
5567 unsigned long pfn, bitidx;
5568 unsigned long flags = 0;
5569 unsigned long value = 1;
5570
5571 zone = page_zone(page);
5572 pfn = page_to_pfn(page);
5573 bitmap = get_pageblock_bitmap(zone, pfn);
5574 bitidx = pfn_to_bitidx(zone, pfn);
5575
5576 for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
5577 if (test_bit(bitidx + start_bitidx, bitmap))
5578 flags |= value;
5579
5580 return flags;
5581}
5582
5583/**
Mel Gormand9c23402007-10-16 01:26:01 -07005584 * set_pageblock_flags_group - Set the requested group of flags for a pageblock_nr_pages block of pages
Mel Gorman835c1342007-10-16 01:25:47 -07005585 * @page: The page within the block of interest
5586 * @start_bitidx: The first bit of interest
5587 * @end_bitidx: The last bit of interest
5588 * @flags: The flags to set
5589 */
5590void set_pageblock_flags_group(struct page *page, unsigned long flags,
5591 int start_bitidx, int end_bitidx)
5592{
5593 struct zone *zone;
5594 unsigned long *bitmap;
5595 unsigned long pfn, bitidx;
5596 unsigned long value = 1;
5597
5598 zone = page_zone(page);
5599 pfn = page_to_pfn(page);
5600 bitmap = get_pageblock_bitmap(zone, pfn);
5601 bitidx = pfn_to_bitidx(zone, pfn);
KAMEZAWA Hiroyuki86051ca2008-04-29 00:58:21 -07005602 VM_BUG_ON(pfn < zone->zone_start_pfn);
5603 VM_BUG_ON(pfn >= zone->zone_start_pfn + zone->spanned_pages);
Mel Gorman835c1342007-10-16 01:25:47 -07005604
5605 for (; start_bitidx <= end_bitidx; start_bitidx++, value <<= 1)
5606 if (flags & value)
5607 __set_bit(bitidx + start_bitidx, bitmap);
5608 else
5609 __clear_bit(bitidx + start_bitidx, bitmap);
5610}
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07005611
5612/*
Minchan Kim13f662c2012-07-31 16:43:01 -07005613 * This function checks whether pageblock includes unmovable pages or not.
5614 * If @count is not zero, it is okay to include less @count unmovable pages
5615 *
5616 * PageLRU check wihtout isolation or lru_lock could race so that
5617 * MIGRATE_MOVABLE block might include unmovable pages. It means you can't
5618 * expect this function should be exact.
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07005619 */
Minchan Kim13f662c2012-07-31 16:43:01 -07005620static bool
5621__has_unmovable_pages(struct zone *zone, struct page *page, int count)
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005622{
5623 unsigned long pfn, iter, found;
Michal Nazarewiczd4158d22011-12-29 13:09:50 +01005624 int mt;
5625
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005626 /*
5627 * For avoiding noise data, lru_add_drain_all() should be called
Minchan Kim13f662c2012-07-31 16:43:01 -07005628 * If ZONE_MOVABLE, the zone never contains unmovable pages
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005629 */
5630 if (zone_idx(zone) == ZONE_MOVABLE)
Minchan Kim13f662c2012-07-31 16:43:01 -07005631 return false;
Michal Nazarewiczd4158d22011-12-29 13:09:50 +01005632 mt = get_pageblock_migratetype(page);
5633 if (mt == MIGRATE_MOVABLE || is_migrate_cma(mt))
Minchan Kim13f662c2012-07-31 16:43:01 -07005634 return false;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005635
5636 pfn = page_to_pfn(page);
5637 for (found = 0, iter = 0; iter < pageblock_nr_pages; iter++) {
5638 unsigned long check = pfn + iter;
5639
Namhyung Kim29723fc2011-02-25 14:44:25 -08005640 if (!pfn_valid_within(check))
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005641 continue;
Namhyung Kim29723fc2011-02-25 14:44:25 -08005642
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005643 page = pfn_to_page(check);
Minchan Kim259ca6a2012-07-31 16:42:59 -07005644 /*
5645 * We can't use page_count without pin a page
5646 * because another CPU can free compound page.
5647 * This check already skips compound tails of THP
5648 * because their page->_count is zero at all time.
5649 */
5650 if (!atomic_read(&page->_count)) {
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005651 if (PageBuddy(page))
5652 iter += (1 << page_order(page)) - 1;
5653 continue;
5654 }
Minchan Kim259ca6a2012-07-31 16:42:59 -07005655
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005656 if (!PageLRU(page))
5657 found++;
5658 /*
5659 * If there are RECLAIMABLE pages, we need to check it.
5660 * But now, memory offline itself doesn't call shrink_slab()
5661 * and it still to be fixed.
5662 */
5663 /*
5664 * If the page is not RAM, page_count()should be 0.
5665 * we don't need more check. This is an _used_ not-movable page.
5666 *
5667 * The problematic thing here is PG_reserved pages. PG_reserved
5668 * is set to both of a memory hole page and a _used_ kernel
5669 * page at boot.
5670 */
5671 if (found > count)
Minchan Kim13f662c2012-07-31 16:43:01 -07005672 return true;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005673 }
Minchan Kim13f662c2012-07-31 16:43:01 -07005674 return false;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005675}
5676
5677bool is_pageblock_removable_nolock(struct page *page)
5678{
Michal Hocko656a0702012-01-20 14:33:58 -08005679 struct zone *zone;
5680 unsigned long pfn;
Michal Hocko687875f2012-01-20 14:33:55 -08005681
5682 /*
5683 * We have to be careful here because we are iterating over memory
5684 * sections which are not zone aware so we might end up outside of
5685 * the zone but still within the section.
Michal Hocko656a0702012-01-20 14:33:58 -08005686 * We have to take care about the node as well. If the node is offline
5687 * its NODE_DATA will be NULL - see page_zone.
Michal Hocko687875f2012-01-20 14:33:55 -08005688 */
Michal Hocko656a0702012-01-20 14:33:58 -08005689 if (!node_online(page_to_nid(page)))
5690 return false;
5691
5692 zone = page_zone(page);
5693 pfn = page_to_pfn(page);
5694 if (zone->zone_start_pfn > pfn ||
Michal Hocko687875f2012-01-20 14:33:55 -08005695 zone->zone_start_pfn + zone->spanned_pages <= pfn)
5696 return false;
5697
Minchan Kim13f662c2012-07-31 16:43:01 -07005698 return !__has_unmovable_pages(zone, page, 0);
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005699}
5700
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07005701int set_migratetype_isolate(struct page *page)
5702{
5703 struct zone *zone;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005704 unsigned long flags, pfn;
Robert Jennings925cc712009-12-17 14:44:38 +00005705 struct memory_isolate_notify arg;
5706 int notifier_ret;
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07005707 int ret = -EBUSY;
5708
5709 zone = page_zone(page);
Robert Jennings925cc712009-12-17 14:44:38 +00005710
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07005711 spin_lock_irqsave(&zone->lock, flags);
Robert Jennings925cc712009-12-17 14:44:38 +00005712
5713 pfn = page_to_pfn(page);
5714 arg.start_pfn = pfn;
5715 arg.nr_pages = pageblock_nr_pages;
5716 arg.pages_found = 0;
5717
5718 /*
5719 * It may be possible to isolate a pageblock even if the
5720 * migratetype is not MIGRATE_MOVABLE. The memory isolation
5721 * notifier chain is used by balloon drivers to return the
5722 * number of pages in a range that are held by the balloon
5723 * driver to shrink memory. If all the pages are accounted for
5724 * by balloons, are free, or on the LRU, isolation can continue.
5725 * Later, for example, when memory hotplug notifier runs, these
5726 * pages reported as "can be isolated" should be isolated(freed)
5727 * by the balloon driver through the memory notifier chain.
5728 */
5729 notifier_ret = memory_isolate_notify(MEM_ISOLATE_COUNT, &arg);
5730 notifier_ret = notifier_to_errno(notifier_ret);
KAMEZAWA Hiroyuki4b204772010-10-26 14:21:29 -07005731 if (notifier_ret)
Robert Jennings925cc712009-12-17 14:44:38 +00005732 goto out;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005733 /*
5734 * FIXME: Now, memory hotplug doesn't call shrink_slab() by itself.
5735 * We just check MOVABLE pages.
5736 */
Minchan Kim13f662c2012-07-31 16:43:01 -07005737 if (!__has_unmovable_pages(zone, page, arg.pages_found))
Robert Jennings925cc712009-12-17 14:44:38 +00005738 ret = 0;
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005739 /*
Minchan Kim13f662c2012-07-31 16:43:01 -07005740 * Unmovable means "not-on-lru" pages. If Unmovable pages are
5741 * larger than removable-by-driver pages reported by notifier,
5742 * we'll fail.
KAMEZAWA Hiroyuki49ac8252010-10-26 14:21:30 -07005743 */
5744
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07005745out:
Robert Jennings925cc712009-12-17 14:44:38 +00005746 if (!ret) {
Larry Bassel6d6e2c92012-12-14 14:21:05 -08005747 unsigned long nr_pages;
5748 int migratetype = get_pageblock_migratetype(page);
5749
Robert Jennings925cc712009-12-17 14:44:38 +00005750 set_pageblock_migratetype(page, MIGRATE_ISOLATE);
Larry Bassel6d6e2c92012-12-14 14:21:05 -08005751 nr_pages = move_freepages_block(zone, page, MIGRATE_ISOLATE);
5752
5753 __mod_zone_freepage_state(zone, -nr_pages, migratetype);
Robert Jennings925cc712009-12-17 14:44:38 +00005754 }
5755
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07005756 spin_unlock_irqrestore(&zone->lock, flags);
5757 if (!ret)
Christoph Lameter9f8f2172008-02-04 22:29:11 -08005758 drain_all_pages();
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07005759 return ret;
5760}
5761
Michal Nazarewiczc80cd922012-04-03 15:06:15 +02005762void unset_migratetype_isolate(struct page *page, unsigned migratetype)
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07005763{
5764 struct zone *zone;
Larry Bassel6d6e2c92012-12-14 14:21:05 -08005765 unsigned long flags, nr_pages;
5766
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07005767 zone = page_zone(page);
5768 spin_lock_irqsave(&zone->lock, flags);
5769 if (get_pageblock_migratetype(page) != MIGRATE_ISOLATE)
5770 goto out;
Larry Bassel6d6e2c92012-12-14 14:21:05 -08005771 nr_pages = move_freepages_block(zone, page, migratetype);
5772 __mod_zone_freepage_state(zone, nr_pages, migratetype);
Michal Nazarewiczc80cd922012-04-03 15:06:15 +02005773 set_pageblock_migratetype(page, migratetype);
KAMEZAWA Hiroyukia5d76b52007-10-16 01:26:11 -07005774out:
5775 spin_unlock_irqrestore(&zone->lock, flags);
5776}
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07005777
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005778#ifdef CONFIG_CMA
5779
5780static unsigned long pfn_max_align_down(unsigned long pfn)
5781{
5782 return pfn & ~(max_t(unsigned long, MAX_ORDER_NR_PAGES,
5783 pageblock_nr_pages) - 1);
5784}
5785
5786static unsigned long pfn_max_align_up(unsigned long pfn)
5787{
5788 return ALIGN(pfn, max_t(unsigned long, MAX_ORDER_NR_PAGES,
5789 pageblock_nr_pages));
5790}
5791
5792static struct page *
5793__alloc_contig_migrate_alloc(struct page *page, unsigned long private,
5794 int **resultp)
5795{
Rabin Vincentfbfcd6f2012-07-05 15:52:23 +05305796 gfp_t gfp_mask = GFP_USER | __GFP_MOVABLE;
5797
5798 if (PageHighMem(page))
5799 gfp_mask |= __GFP_HIGHMEM;
5800
5801 return alloc_page(gfp_mask);
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005802}
5803
5804/* [start, end) must belong to a single zone. */
Mel Gorman6b506642012-10-08 16:32:41 -07005805static int __alloc_contig_migrate_range(struct compact_control *cc,
5806 unsigned long start, unsigned long end)
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005807{
5808 /* This function is based on compact_zone() from compaction.c. */
Minchan Kim726cfb42012-10-08 16:33:51 -07005809 unsigned long nr_reclaimed;
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005810 unsigned long pfn = start;
5811 unsigned int tries = 0;
5812 int ret = 0;
5813
woojoong.lee539118f2012-12-03 17:58:43 -08005814 migrate_prep();
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005815
Mel Gorman6b506642012-10-08 16:32:41 -07005816 while (pfn < end || !list_empty(&cc->migratepages)) {
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005817 if (fatal_signal_pending(current)) {
5818 ret = -EINTR;
5819 break;
5820 }
5821
Mel Gorman6b506642012-10-08 16:32:41 -07005822 if (list_empty(&cc->migratepages)) {
5823 cc->nr_migratepages = 0;
5824 pfn = isolate_migratepages_range(cc->zone, cc,
Minchan Kimd05b4522012-10-08 16:33:48 -07005825 pfn, end, true);
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005826 if (!pfn) {
5827 ret = -EINTR;
5828 break;
5829 }
5830 tries = 0;
5831 } else if (++tries == 5) {
5832 ret = ret < 0 ? ret : -EBUSY;
5833 break;
5834 }
5835
Mel Gorman6b506642012-10-08 16:32:41 -07005836 nr_reclaimed = reclaim_clean_pages_from_list(cc->zone, &cc->migratepages);
Minchan Kimaab50f12012-10-08 16:31:55 -07005837
Mel Gorman6b506642012-10-08 16:32:41 -07005838 cc->nr_migratepages -= nr_reclaimed;
5839
5840 ret = migrate_pages(&cc->migratepages,
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005841 __alloc_contig_migrate_alloc,
Minchan Kim132fb792012-05-11 09:37:13 +02005842 0, false, MIGRATE_SYNC);
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005843 }
5844
Mel Gorman6b506642012-10-08 16:32:41 -07005845 putback_lru_pages(&cc->migratepages);
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005846 return ret > 0 ? 0 : ret;
5847}
5848
5849/**
5850 * alloc_contig_range() -- tries to allocate given range of pages
5851 * @start: start PFN to allocate
5852 * @end: one-past-the-last PFN to allocate
Michal Nazarewiczc80cd922012-04-03 15:06:15 +02005853 * @migratetype: migratetype of the underlaying pageblocks (either
5854 * #MIGRATE_MOVABLE or #MIGRATE_CMA). All pageblocks
5855 * in range must have the same migratetype and it must
5856 * be either of the two.
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005857 *
5858 * The PFN range does not have to be pageblock or MAX_ORDER_NR_PAGES
5859 * aligned, however it's the caller's responsibility to guarantee that
5860 * we are the only thread that changes migrate type of pageblocks the
5861 * pages fall in.
5862 *
5863 * The PFN range must belong to a single zone.
5864 *
5865 * Returns zero on success or negative error code. On success all
5866 * pages which PFN is in [start, end) are allocated for the caller and
5867 * need to be freed with free_contig_range().
5868 */
Michal Nazarewiczc80cd922012-04-03 15:06:15 +02005869int alloc_contig_range(unsigned long start, unsigned long end,
5870 unsigned migratetype)
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005871{
5872 struct zone *zone = page_zone(pfn_to_page(start));
5873 unsigned long outer_start, outer_end;
5874 int ret = 0, order;
5875
Mel Gorman6b506642012-10-08 16:32:41 -07005876 struct compact_control cc = {
5877 .nr_migratepages = 0,
5878 .order = -1,
5879 .zone = page_zone(pfn_to_page(start)),
5880 .sync = true,
5881 .ignore_skip_hint = true,
5882 };
5883 INIT_LIST_HEAD(&cc.migratepages);
5884
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005885 /*
5886 * What we do here is we mark all pageblocks in range as
5887 * MIGRATE_ISOLATE. Because pageblock and max order pages may
5888 * have different sizes, and due to the way page allocator
5889 * work, we align the range to biggest of the two pages so
5890 * that page allocator won't try to merge buddies from
5891 * different pageblocks and change MIGRATE_ISOLATE to some
5892 * other migration type.
5893 *
5894 * Once the pageblocks are marked as MIGRATE_ISOLATE, we
5895 * migrate the pages from an unaligned range (ie. pages that
5896 * we are interested in). This will put all the pages in
5897 * range back to page allocator as MIGRATE_ISOLATE.
5898 *
5899 * When this is done, we take the pages in range from page
5900 * allocator removing them from the buddy system. This way
5901 * page allocator will never consider using them.
5902 *
5903 * This lets us mark the pageblocks back as
5904 * MIGRATE_CMA/MIGRATE_MOVABLE so that free pages in the
5905 * aligned range but not in the unaligned, original range are
5906 * put back to page allocator so that buddy can use them.
5907 */
5908
5909 ret = start_isolate_page_range(pfn_max_align_down(start),
Michal Nazarewiczc80cd922012-04-03 15:06:15 +02005910 pfn_max_align_up(end), migratetype);
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005911 if (ret)
5912 goto done;
5913
Heesub Shin771aaa62013-01-07 11:10:13 +09005914 zone->cma_alloc = 1;
5915
Mel Gorman6b506642012-10-08 16:32:41 -07005916 ret = __alloc_contig_migrate_range(&cc, start, end);
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005917 if (ret)
5918 goto done;
5919
5920 /*
5921 * Pages from [start, end) are within a MAX_ORDER_NR_PAGES
5922 * aligned blocks that are marked as MIGRATE_ISOLATE. What's
5923 * more, all pages in [start, end) are free in page allocator.
5924 * What we are going to do is to allocate all pages from
5925 * [start, end) (that is remove them from page allocator).
5926 *
5927 * The only problem is that pages at the beginning and at the
5928 * end of interesting range may be not aligned with pages that
5929 * page allocator holds, ie. they can be part of higher order
5930 * pages. Because of this, we reserve the bigger range and
5931 * once this is done free the pages we are not interested in.
5932 *
5933 * We don't have to hold zone->lock here because the pages are
5934 * isolated thus they won't get removed from buddy.
5935 */
5936
5937 lru_add_drain_all();
5938 drain_all_pages();
5939
5940 order = 0;
5941 outer_start = start;
5942 while (!PageBuddy(pfn_to_page(outer_start))) {
5943 if (++order >= MAX_ORDER) {
5944 ret = -EBUSY;
5945 goto done;
5946 }
5947 outer_start &= ~0UL << order;
5948 }
5949
5950 /* Make sure the range is really isolated. */
5951 if (test_pages_isolated(outer_start, end)) {
5952 pr_warn("alloc_contig_range test_pages_isolated(%lx, %lx) failed\n",
5953 outer_start, end);
5954 ret = -EBUSY;
5955 goto done;
5956 }
5957
Marek Szyprowskif1f63882012-01-25 12:49:24 +01005958
5959 /* Grab isolated pages from freelists. */
Mel Gorman6b506642012-10-08 16:32:41 -07005960 outer_end = isolate_freepages_range(&cc, outer_start, end);
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005961 if (!outer_end) {
5962 ret = -EBUSY;
5963 goto done;
5964 }
5965
5966 /* Free head and tail (if any) */
5967 if (start != outer_start)
5968 free_contig_range(outer_start, start - outer_start);
5969 if (end != outer_end)
5970 free_contig_range(end, outer_end - end);
5971
5972done:
5973 undo_isolate_page_range(pfn_max_align_down(start),
Michal Nazarewiczc80cd922012-04-03 15:06:15 +02005974 pfn_max_align_up(end), migratetype);
Heesub Shin771aaa62013-01-07 11:10:13 +09005975 zone->cma_alloc = 0;
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005976 return ret;
5977}
5978
5979void free_contig_range(unsigned long pfn, unsigned nr_pages)
5980{
Marek Szyprowskidad0d8b2012-12-20 15:05:18 -08005981 unsigned int count = 0;
5982
5983 for (; nr_pages--; pfn++) {
5984 struct page *page = pfn_to_page(pfn);
5985
5986 count += page_count(page) != 1;
5987 __free_page(page);
5988 }
5989 WARN(count != 0, "%d pages are still in use!\n", count);
Michal Nazarewicz4c1ff372011-12-29 13:09:50 +01005990}
5991#endif
5992
KAMEZAWA Hiroyuki0c0e6192007-10-16 01:26:12 -07005993#ifdef CONFIG_MEMORY_HOTREMOVE
5994/*
5995 * All pages in the range must be isolated before calling this.
5996 */
5997void
5998__offline_isolated_pages(unsigned long start_pfn, unsigned long end_pfn)
5999{
6000 struct page *page;
6001 struct zone *zone;
6002 int order, i;
6003 unsigned long pfn;
6004 unsigned long flags;
6005 /* find the first valid pfn */
6006 for (pfn = start_pfn; pfn < end_pfn; pfn++)
6007 if (pfn_valid(pfn))
6008 break;
6009 if (pfn == end_pfn)
6010 return;
6011 zone = page_zone(pfn_to_page(pfn));
6012 spin_lock_irqsave(&zone->lock, flags);
6013 pfn = start_pfn;
6014 while (pfn < end_pfn) {
6015 if (!pfn_valid(pfn)) {
6016 pfn++;
6017 continue;
6018 }
6019 page = pfn_to_page(pfn);
6020 BUG_ON(page_count(page));
6021 BUG_ON(!PageBuddy(page));
6022 order = page_order(page);
6023#ifdef CONFIG_DEBUG_VM
6024 printk(KERN_INFO "remove from free list %lx %d %lx\n",
6025 pfn, 1 << order, end_pfn);
6026#endif
6027 list_del(&page->lru);
6028 rmv_page_order(page);
6029 zone->free_area[order].nr_free--;
6030 __mod_zone_page_state(zone, NR_FREE_PAGES,
6031 - (1UL << order));
6032 for (i = 0; i < (1 << order); i++)
6033 SetPageReserved((page+i));
6034 pfn += (1 << order);
6035 }
6036 spin_unlock_irqrestore(&zone->lock, flags);
6037}
6038#endif
Wu Fengguang8d22ba12009-12-16 12:19:58 +01006039
6040#ifdef CONFIG_MEMORY_FAILURE
6041bool is_free_buddy_page(struct page *page)
6042{
6043 struct zone *zone = page_zone(page);
6044 unsigned long pfn = page_to_pfn(page);
6045 unsigned long flags;
6046 int order;
6047
6048 spin_lock_irqsave(&zone->lock, flags);
6049 for (order = 0; order < MAX_ORDER; order++) {
6050 struct page *page_head = page - (pfn & ((1 << order) - 1));
6051
6052 if (PageBuddy(page_head) && page_order(page_head) >= order)
6053 break;
6054 }
6055 spin_unlock_irqrestore(&zone->lock, flags);
6056
6057 return order < MAX_ORDER;
6058}
6059#endif
Wu Fengguang718a3822010-03-10 15:20:43 -08006060
6061static struct trace_print_flags pageflag_names[] = {
6062 {1UL << PG_locked, "locked" },
6063 {1UL << PG_error, "error" },
6064 {1UL << PG_referenced, "referenced" },
6065 {1UL << PG_uptodate, "uptodate" },
6066 {1UL << PG_dirty, "dirty" },
6067 {1UL << PG_lru, "lru" },
6068 {1UL << PG_active, "active" },
6069 {1UL << PG_slab, "slab" },
6070 {1UL << PG_owner_priv_1, "owner_priv_1" },
6071 {1UL << PG_arch_1, "arch_1" },
6072 {1UL << PG_reserved, "reserved" },
6073 {1UL << PG_private, "private" },
6074 {1UL << PG_private_2, "private_2" },
6075 {1UL << PG_writeback, "writeback" },
6076#ifdef CONFIG_PAGEFLAGS_EXTENDED
6077 {1UL << PG_head, "head" },
6078 {1UL << PG_tail, "tail" },
6079#else
6080 {1UL << PG_compound, "compound" },
6081#endif
6082 {1UL << PG_swapcache, "swapcache" },
6083 {1UL << PG_mappedtodisk, "mappedtodisk" },
6084 {1UL << PG_reclaim, "reclaim" },
Wu Fengguang718a3822010-03-10 15:20:43 -08006085 {1UL << PG_swapbacked, "swapbacked" },
6086 {1UL << PG_unevictable, "unevictable" },
6087#ifdef CONFIG_MMU
6088 {1UL << PG_mlocked, "mlocked" },
6089#endif
6090#ifdef CONFIG_ARCH_USES_PG_UNCACHED
6091 {1UL << PG_uncached, "uncached" },
6092#endif
6093#ifdef CONFIG_MEMORY_FAILURE
6094 {1UL << PG_hwpoison, "hwpoison" },
6095#endif
Lee Susman0cc1a0c2013-05-05 17:31:17 +03006096 {1UL << PG_readahead, "PG_readahead" },
Wu Fengguang718a3822010-03-10 15:20:43 -08006097 {-1UL, NULL },
6098};
6099
6100static void dump_page_flags(unsigned long flags)
6101{
6102 const char *delim = "";
6103 unsigned long mask;
6104 int i;
6105
6106 printk(KERN_ALERT "page flags: %#lx(", flags);
6107
6108 /* remove zone id */
6109 flags &= (1UL << NR_PAGEFLAGS) - 1;
6110
6111 for (i = 0; pageflag_names[i].name && flags; i++) {
6112
6113 mask = pageflag_names[i].mask;
6114 if ((flags & mask) != mask)
6115 continue;
6116
6117 flags &= ~mask;
6118 printk("%s%s", delim, pageflag_names[i].name);
6119 delim = "|";
6120 }
6121
6122 /* check for left over flags */
6123 if (flags)
6124 printk("%s%#lx", delim, flags);
6125
6126 printk(")\n");
6127}
6128
6129void dump_page(struct page *page)
6130{
6131 printk(KERN_ALERT
6132 "page:%p count:%d mapcount:%d mapping:%p index:%#lx\n",
Andrea Arcangeli4e9f64c2011-01-13 15:46:29 -08006133 page, atomic_read(&page->_count), page_mapcount(page),
Wu Fengguang718a3822010-03-10 15:20:43 -08006134 page->mapping, page->index);
6135 dump_page_flags(page->flags);
Daisuke Nishimuraf212ad72011-03-23 16:42:25 -07006136 mem_cgroup_print_bad_page(page);
Wu Fengguang718a3822010-03-10 15:20:43 -08006137}