blob: 09c14e213b63eb72f2f614695a886e67da7444d8 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef _LINUX_MMZONE_H
2#define _LINUX_MMZONE_H
3
Linus Torvalds1da177e2005-04-16 15:20:36 -07004#ifndef __ASSEMBLY__
Christoph Lameter97965472008-04-28 02:12:54 -07005#ifndef __GENERATING_BOUNDS_H
Linus Torvalds1da177e2005-04-16 15:20:36 -07006
Linus Torvalds1da177e2005-04-16 15:20:36 -07007#include <linux/spinlock.h>
8#include <linux/list.h>
9#include <linux/wait.h>
David Rientjese815af92007-10-16 23:25:54 -070010#include <linux/bitops.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/cache.h>
12#include <linux/threads.h>
13#include <linux/numa.h>
14#include <linux/init.h>
Dave Hansenbdc8cb92005-10-29 18:16:53 -070015#include <linux/seqlock.h>
KAMEZAWA Hiroyuki8357f862006-03-27 01:15:57 -080016#include <linux/nodemask.h>
Mel Gorman835c1342007-10-16 01:25:47 -070017#include <linux/pageblock-flags.h>
Christoph Lameter97965472008-04-28 02:12:54 -070018#include <linux/bounds.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070019#include <asm/atomic.h>
Ralf Baechle93ff66b2006-06-04 02:51:29 -070020#include <asm/page.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070021
22/* Free memory management - zoned buddy allocator. */
23#ifndef CONFIG_FORCE_MAX_ZONEORDER
24#define MAX_ORDER 11
25#else
26#define MAX_ORDER CONFIG_FORCE_MAX_ZONEORDER
27#endif
Bob Piccoe984bb42006-05-20 15:00:31 -070028#define MAX_ORDER_NR_PAGES (1 << (MAX_ORDER - 1))
Linus Torvalds1da177e2005-04-16 15:20:36 -070029
Andy Whitcroft5ad333e2007-07-17 04:03:16 -070030/*
31 * PAGE_ALLOC_COSTLY_ORDER is the order at which allocations are deemed
32 * costly to service. That is between allocation orders which should
33 * coelesce naturally under reasonable reclaim pressure and those which
34 * will not.
35 */
36#define PAGE_ALLOC_COSTLY_ORDER 3
37
Mel Gormanb2a0ac82007-10-16 01:25:48 -070038#define MIGRATE_UNMOVABLE 0
Mel Gormane12ba742007-10-16 01:25:52 -070039#define MIGRATE_RECLAIMABLE 1
40#define MIGRATE_MOVABLE 2
Mel Gorman64c5e132007-10-16 01:25:59 -070041#define MIGRATE_RESERVE 3
KAMEZAWA Hiroyukia5d76b542007-10-16 01:26:11 -070042#define MIGRATE_ISOLATE 4 /* can't allocate from here */
43#define MIGRATE_TYPES 5
Mel Gormanb2a0ac82007-10-16 01:25:48 -070044
45#define for_each_migratetype_order(order, type) \
46 for (order = 0; order < MAX_ORDER; order++) \
47 for (type = 0; type < MIGRATE_TYPES; type++)
48
Mel Gorman467c9962007-10-16 01:26:02 -070049extern int page_group_by_mobility_disabled;
50
51static inline int get_pageblock_migratetype(struct page *page)
52{
53 if (unlikely(page_group_by_mobility_disabled))
54 return MIGRATE_UNMOVABLE;
55
56 return get_pageblock_flags_group(page, PB_migrate, PB_migrate_end);
57}
58
Linus Torvalds1da177e2005-04-16 15:20:36 -070059struct free_area {
Mel Gormanb2a0ac82007-10-16 01:25:48 -070060 struct list_head free_list[MIGRATE_TYPES];
Linus Torvalds1da177e2005-04-16 15:20:36 -070061 unsigned long nr_free;
62};
63
64struct pglist_data;
65
66/*
67 * zone->lock and zone->lru_lock are two of the hottest locks in the kernel.
68 * So add a wild amount of padding here to ensure that they fall into separate
69 * cachelines. There are very few zone structures in the machine, so space
70 * consumption is not a concern here.
71 */
72#if defined(CONFIG_SMP)
73struct zone_padding {
74 char x[0];
Ravikiran G Thirumalai22fc6ec2006-01-08 01:01:27 -080075} ____cacheline_internodealigned_in_smp;
Linus Torvalds1da177e2005-04-16 15:20:36 -070076#define ZONE_PADDING(name) struct zone_padding name;
77#else
78#define ZONE_PADDING(name)
79#endif
80
Christoph Lameter2244b952006-06-30 01:55:33 -070081enum zone_stat_item {
Christoph Lameter51ed4492007-02-10 01:43:02 -080082 /* First 128 byte cacheline (assuming 64 bit words) */
Christoph Lameterd23ad422007-02-10 01:43:02 -080083 NR_FREE_PAGES,
Christoph Lameterb69408e2008-10-18 20:26:14 -070084 NR_LRU_BASE,
Rik van Riel4f98a2f2008-10-18 20:26:32 -070085 NR_INACTIVE_ANON = NR_LRU_BASE, /* must match order of LRU_[IN]ACTIVE */
86 NR_ACTIVE_ANON, /* " " " " " */
87 NR_INACTIVE_FILE, /* " " " " " */
88 NR_ACTIVE_FILE, /* " " " " " */
Lee Schermerhorn894bc312008-10-18 20:26:39 -070089#ifdef CONFIG_UNEVICTABLE_LRU
90 NR_UNEVICTABLE, /* " " " " " */
Nick Piggin5344b7e2008-10-18 20:26:51 -070091 NR_MLOCK, /* mlock()ed pages found and moved off LRU */
Lee Schermerhorn894bc312008-10-18 20:26:39 -070092#else
93 NR_UNEVICTABLE = NR_ACTIVE_FILE, /* avoid compiler errors in dead code */
Nick Piggin5344b7e2008-10-18 20:26:51 -070094 NR_MLOCK = NR_ACTIVE_FILE,
Lee Schermerhorn894bc312008-10-18 20:26:39 -070095#endif
Christoph Lameterf3dbd342006-06-30 01:55:36 -070096 NR_ANON_PAGES, /* Mapped anonymous pages */
97 NR_FILE_MAPPED, /* pagecache pages mapped into pagetables.
Christoph Lameter65ba55f2006-06-30 01:55:34 -070098 only modified from process context */
Christoph Lameter347ce432006-06-30 01:55:35 -070099 NR_FILE_PAGES,
Christoph Lameterb1e7a8f2006-06-30 01:55:39 -0700100 NR_FILE_DIRTY,
Christoph Lameterce866b32006-06-30 01:55:40 -0700101 NR_WRITEBACK,
Christoph Lameter51ed4492007-02-10 01:43:02 -0800102 NR_SLAB_RECLAIMABLE,
103 NR_SLAB_UNRECLAIMABLE,
104 NR_PAGETABLE, /* used for pagetables */
Christoph Lameterfd39fc82006-06-30 01:55:40 -0700105 NR_UNSTABLE_NFS, /* NFS unstable pages */
Christoph Lameterd2c5e302006-06-30 01:55:41 -0700106 NR_BOUNCE,
Andrew Mortone129b5c2006-09-27 01:50:00 -0700107 NR_VMSCAN_WRITE,
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700108 /* Second 128 byte cacheline */
Miklos Szeredifc3ba692008-04-30 00:54:38 -0700109 NR_WRITEBACK_TEMP, /* Writeback using temporary buffers */
Christoph Lameterca889e62006-06-30 01:55:44 -0700110#ifdef CONFIG_NUMA
111 NUMA_HIT, /* allocated in intended node */
112 NUMA_MISS, /* allocated in non intended node */
113 NUMA_FOREIGN, /* was intended here, hit elsewhere */
114 NUMA_INTERLEAVE_HIT, /* interleaver preferred this zone */
115 NUMA_LOCAL, /* allocation from local node */
116 NUMA_OTHER, /* allocation from other node */
117#endif
Christoph Lameter2244b952006-06-30 01:55:33 -0700118 NR_VM_ZONE_STAT_ITEMS };
119
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700120/*
121 * We do arithmetic on the LRU lists in various places in the code,
122 * so it is important to keep the active lists LRU_ACTIVE higher in
123 * the array than the corresponding inactive lists, and to keep
124 * the *_FILE lists LRU_FILE higher than the corresponding _ANON lists.
125 *
126 * This has to be kept in sync with the statistics in zone_stat_item
127 * above and the descriptions in vmstat_text in mm/vmstat.c
128 */
129#define LRU_BASE 0
130#define LRU_ACTIVE 1
131#define LRU_FILE 2
132
Christoph Lameterb69408e2008-10-18 20:26:14 -0700133enum lru_list {
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700134 LRU_INACTIVE_ANON = LRU_BASE,
135 LRU_ACTIVE_ANON = LRU_BASE + LRU_ACTIVE,
136 LRU_INACTIVE_FILE = LRU_BASE + LRU_FILE,
137 LRU_ACTIVE_FILE = LRU_BASE + LRU_FILE + LRU_ACTIVE,
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700138#ifdef CONFIG_UNEVICTABLE_LRU
139 LRU_UNEVICTABLE,
140#else
141 LRU_UNEVICTABLE = LRU_ACTIVE_FILE, /* avoid compiler errors in dead code */
142#endif
143 NR_LRU_LISTS
144};
Christoph Lameterb69408e2008-10-18 20:26:14 -0700145
146#define for_each_lru(l) for (l = 0; l < NR_LRU_LISTS; l++)
147
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700148#define for_each_evictable_lru(l) for (l = 0; l <= LRU_ACTIVE_FILE; l++)
149
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700150static inline int is_file_lru(enum lru_list l)
151{
152 return (l == LRU_INACTIVE_FILE || l == LRU_ACTIVE_FILE);
153}
154
Christoph Lameterb69408e2008-10-18 20:26:14 -0700155static inline int is_active_lru(enum lru_list l)
156{
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700157 return (l == LRU_ACTIVE_ANON || l == LRU_ACTIVE_FILE);
Christoph Lameterb69408e2008-10-18 20:26:14 -0700158}
159
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700160static inline int is_unevictable_lru(enum lru_list l)
161{
162#ifdef CONFIG_UNEVICTABLE_LRU
163 return (l == LRU_UNEVICTABLE);
164#else
165 return 0;
166#endif
167}
168
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169struct per_cpu_pages {
170 int count; /* number of pages in the list */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171 int high; /* high watermark, emptying needed */
172 int batch; /* chunk size for buddy add/remove */
173 struct list_head list; /* the list of pages */
174};
175
176struct per_cpu_pageset {
Christoph Lameter3dfa5722008-02-04 22:29:19 -0800177 struct per_cpu_pages pcp;
Christoph Lameter4037d452007-05-09 02:35:14 -0700178#ifdef CONFIG_NUMA
179 s8 expire;
180#endif
Christoph Lameter2244b952006-06-30 01:55:33 -0700181#ifdef CONFIG_SMP
Christoph Lameterdf9ecab2006-08-31 21:27:35 -0700182 s8 stat_threshold;
Christoph Lameter2244b952006-06-30 01:55:33 -0700183 s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS];
184#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700185} ____cacheline_aligned_in_smp;
186
Christoph Lametere7c8d5c2005-06-21 17:14:47 -0700187#ifdef CONFIG_NUMA
188#define zone_pcp(__z, __cpu) ((__z)->pageset[(__cpu)])
189#else
190#define zone_pcp(__z, __cpu) (&(__z)->pageset[(__cpu)])
191#endif
192
Christoph Lameter97965472008-04-28 02:12:54 -0700193#endif /* !__GENERATING_BOUNDS.H */
194
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700195enum zone_type {
Christoph Lameter4b51d662007-02-10 01:43:10 -0800196#ifdef CONFIG_ZONE_DMA
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700197 /*
198 * ZONE_DMA is used when there are devices that are not able
199 * to do DMA to all of addressable memory (ZONE_NORMAL). Then we
200 * carve out the portion of memory that is needed for these devices.
201 * The range is arch specific.
202 *
203 * Some examples
204 *
205 * Architecture Limit
206 * ---------------------------
207 * parisc, ia64, sparc <4G
208 * s390 <2G
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700209 * arm Various
210 * alpha Unlimited or 0-16MB.
211 *
212 * i386, x86_64 and multiple other arches
213 * <16M.
214 */
215 ZONE_DMA,
Christoph Lameter4b51d662007-02-10 01:43:10 -0800216#endif
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700217#ifdef CONFIG_ZONE_DMA32
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700218 /*
219 * x86_64 needs two ZONE_DMAs because it supports devices that are
220 * only able to do DMA to the lower 16M but also 32 bit devices that
221 * can only do DMA areas below 4G.
222 */
223 ZONE_DMA32,
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700224#endif
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700225 /*
226 * Normal addressable memory is in ZONE_NORMAL. DMA operations can be
227 * performed on pages in ZONE_NORMAL if the DMA devices support
228 * transfers to all addressable memory.
229 */
230 ZONE_NORMAL,
Christoph Lametere53ef382006-09-25 23:31:14 -0700231#ifdef CONFIG_HIGHMEM
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700232 /*
233 * A memory area that is only addressable by the kernel through
234 * mapping portions into its own address space. This is for example
235 * used by i386 to allow the kernel to address the memory beyond
236 * 900MB. The kernel will set up special mappings (page
237 * table entries on i386) for each page that the kernel needs to
238 * access.
239 */
240 ZONE_HIGHMEM,
Christoph Lametere53ef382006-09-25 23:31:14 -0700241#endif
Mel Gorman2a1e2742007-07-17 04:03:12 -0700242 ZONE_MOVABLE,
Christoph Lameter97965472008-04-28 02:12:54 -0700243 __MAX_NR_ZONES
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700244};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700245
Christoph Lameter97965472008-04-28 02:12:54 -0700246#ifndef __GENERATING_BOUNDS_H
247
Linus Torvalds1da177e2005-04-16 15:20:36 -0700248/*
249 * When a memory allocation must conform to specific limitations (such
250 * as being suitable for DMA) the caller will pass in hints to the
251 * allocator in the gfp_mask, in the zone modifier bits. These bits
252 * are used to select a priority ordered list of memory zones which
Christoph Lameter19655d32006-09-25 23:31:19 -0700253 * match the requested limits. See gfp_zone() in include/linux/gfp.h
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254 */
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700255
Christoph Lameter97965472008-04-28 02:12:54 -0700256#if MAX_NR_ZONES < 2
Christoph Lameter4b51d662007-02-10 01:43:10 -0800257#define ZONES_SHIFT 0
Christoph Lameter97965472008-04-28 02:12:54 -0700258#elif MAX_NR_ZONES <= 2
Christoph Lameter19655d32006-09-25 23:31:19 -0700259#define ZONES_SHIFT 1
Christoph Lameter97965472008-04-28 02:12:54 -0700260#elif MAX_NR_ZONES <= 4
Christoph Lameter19655d32006-09-25 23:31:19 -0700261#define ZONES_SHIFT 2
Christoph Lameter4b51d662007-02-10 01:43:10 -0800262#else
263#error ZONES_SHIFT -- too many zones configured adjust calculation
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700264#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700265
KOSAKI Motohiro6e901572009-01-07 18:08:15 -0800266struct zone_reclaim_stat {
267 /*
268 * The pageout code in vmscan.c keeps track of how many of the
269 * mem/swap backed and file backed pages are refeferenced.
270 * The higher the rotated/scanned ratio, the more valuable
271 * that cache is.
272 *
273 * The anon LRU stats live in [0], file LRU stats in [1]
274 */
275 unsigned long recent_rotated[2];
276 unsigned long recent_scanned[2];
277};
278
Linus Torvalds1da177e2005-04-16 15:20:36 -0700279struct zone {
280 /* Fields commonly accessed by the page allocator */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700281 unsigned long pages_min, pages_low, pages_high;
282 /*
283 * We don't know if the memory that we're going to allocate will be freeable
284 * or/and it will be released eventually, so to avoid totally wasting several
285 * GB of ram we must reserve some of the lower zone memory (otherwise we risk
286 * to run OOM on the lower zones despite there's tons of freeable ram
287 * on the higher zones). This array is recalculated at runtime if the
288 * sysctl_lowmem_reserve_ratio sysctl changes.
289 */
290 unsigned long lowmem_reserve[MAX_NR_ZONES];
291
Christoph Lametere7c8d5c2005-06-21 17:14:47 -0700292#ifdef CONFIG_NUMA
Christoph Lameterd5f541e2006-09-27 01:50:08 -0700293 int node;
Christoph Lameter96146342006-07-03 00:24:13 -0700294 /*
295 * zone reclaim becomes active if more unmapped pages exist.
296 */
Christoph Lameter8417bba2006-09-25 23:31:51 -0700297 unsigned long min_unmapped_pages;
Christoph Lameter0ff38492006-09-25 23:31:52 -0700298 unsigned long min_slab_pages;
Christoph Lametere7c8d5c2005-06-21 17:14:47 -0700299 struct per_cpu_pageset *pageset[NR_CPUS];
300#else
Linus Torvalds1da177e2005-04-16 15:20:36 -0700301 struct per_cpu_pageset pageset[NR_CPUS];
Christoph Lametere7c8d5c2005-06-21 17:14:47 -0700302#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 /*
304 * free areas of different sizes
305 */
306 spinlock_t lock;
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700307#ifdef CONFIG_MEMORY_HOTPLUG
308 /* see spanned/present_pages for more description */
309 seqlock_t span_seqlock;
310#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700311 struct free_area free_area[MAX_ORDER];
312
Mel Gorman835c1342007-10-16 01:25:47 -0700313#ifndef CONFIG_SPARSEMEM
314 /*
Mel Gormand9c23402007-10-16 01:26:01 -0700315 * Flags for a pageblock_nr_pages block. See pageblock-flags.h.
Mel Gorman835c1342007-10-16 01:25:47 -0700316 * In SPARSEMEM, this map is stored in struct mem_section
317 */
318 unsigned long *pageblock_flags;
319#endif /* CONFIG_SPARSEMEM */
320
Linus Torvalds1da177e2005-04-16 15:20:36 -0700321
322 ZONE_PADDING(_pad1_)
323
324 /* Fields commonly accessed by the page reclaim scanner */
325 spinlock_t lru_lock;
Christoph Lameterb69408e2008-10-18 20:26:14 -0700326 struct {
327 struct list_head list;
328 unsigned long nr_scan;
329 } lru[NR_LRU_LISTS];
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700330
KOSAKI Motohiro6e901572009-01-07 18:08:15 -0800331 struct zone_reclaim_stat reclaim_stat;
Rik van Riel4f98a2f2008-10-18 20:26:32 -0700332
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333 unsigned long pages_scanned; /* since last reclaim */
David Rientjese815af92007-10-16 23:25:54 -0700334 unsigned long flags; /* zone flags, see below */
Martin Hicks753ee722005-06-21 17:14:41 -0700335
Christoph Lameter2244b952006-06-30 01:55:33 -0700336 /* Zone statistics */
337 atomic_long_t vm_stat[NR_VM_ZONE_STAT_ITEMS];
Christoph Lameter9eeff232006-01-18 17:42:31 -0800338
339 /*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340 * prev_priority holds the scanning priority for this zone. It is
341 * defined as the scanning priority at which we achieved our reclaim
342 * target at the previous try_to_free_pages() or balance_pgdat()
343 * invokation.
344 *
345 * We use prev_priority as a measure of how much stress page reclaim is
346 * under - it drives the swappiness decision: whether to unmap mapped
347 * pages.
348 *
Martin Bligh3bb1a8522006-10-28 10:38:24 -0700349 * Access to both this field is quite racy even on uniprocessor. But
Linus Torvalds1da177e2005-04-16 15:20:36 -0700350 * it is expected to average out OK.
351 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352 int prev_priority;
353
Rik van Riel556adec2008-10-18 20:26:34 -0700354 /*
355 * The target ratio of ACTIVE_ANON to INACTIVE_ANON pages on
356 * this zone's LRU. Maintained by the pageout code.
357 */
358 unsigned int inactive_ratio;
359
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360
361 ZONE_PADDING(_pad2_)
362 /* Rarely used or read-mostly fields */
363
364 /*
365 * wait_table -- the array holding the hash table
Yasunori Goto02b694d2006-06-23 02:03:08 -0700366 * wait_table_hash_nr_entries -- the size of the hash table array
Linus Torvalds1da177e2005-04-16 15:20:36 -0700367 * wait_table_bits -- wait_table_size == (1 << wait_table_bits)
368 *
369 * The purpose of all these is to keep track of the people
370 * waiting for a page to become available and make them
371 * runnable again when possible. The trouble is that this
372 * consumes a lot of space, especially when so few things
373 * wait on pages at a given time. So instead of using
374 * per-page waitqueues, we use a waitqueue hash table.
375 *
376 * The bucket discipline is to sleep on the same queue when
377 * colliding and wake all in that wait queue when removing.
378 * When something wakes, it must check to be sure its page is
379 * truly available, a la thundering herd. The cost of a
380 * collision is great, but given the expected load of the
381 * table, they should be so rare as to be outweighed by the
382 * benefits from the saved space.
383 *
384 * __wait_on_page_locked() and unlock_page() in mm/filemap.c, are the
385 * primary users of these fields, and in mm/page_alloc.c
386 * free_area_init_core() performs the initialization of them.
387 */
388 wait_queue_head_t * wait_table;
Yasunori Goto02b694d2006-06-23 02:03:08 -0700389 unsigned long wait_table_hash_nr_entries;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 unsigned long wait_table_bits;
391
392 /*
393 * Discontig memory support fields.
394 */
395 struct pglist_data *zone_pgdat;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 /* zone_start_pfn == zone_start_paddr >> PAGE_SHIFT */
397 unsigned long zone_start_pfn;
398
Dave Hansenbdc8cb92005-10-29 18:16:53 -0700399 /*
400 * zone_start_pfn, spanned_pages and present_pages are all
401 * protected by span_seqlock. It is a seqlock because it has
402 * to be read outside of zone->lock, and it is done in the main
403 * allocator path. But, it is written quite infrequently.
404 *
405 * The lock is declared along with zone->lock because it is
406 * frequently read in proximity to zone->lock. It's good to
407 * give them a chance of being in the same cacheline.
408 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700409 unsigned long spanned_pages; /* total size, including holes */
410 unsigned long present_pages; /* amount of memory (excluding holes) */
411
412 /*
413 * rarely used fields:
414 */
Helge Deller15ad7cd2006-12-06 20:40:36 -0800415 const char *name;
Ravikiran G Thirumalai22fc6ec2006-01-08 01:01:27 -0800416} ____cacheline_internodealigned_in_smp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700417
David Rientjese815af92007-10-16 23:25:54 -0700418typedef enum {
419 ZONE_ALL_UNRECLAIMABLE, /* all pages pinned */
420 ZONE_RECLAIM_LOCKED, /* prevents concurrent reclaim */
David Rientjes098d7f12007-10-16 23:25:55 -0700421 ZONE_OOM_LOCKED, /* zone is in OOM killer zonelist */
David Rientjese815af92007-10-16 23:25:54 -0700422} zone_flags_t;
423
424static inline void zone_set_flag(struct zone *zone, zone_flags_t flag)
425{
426 set_bit(flag, &zone->flags);
427}
David Rientjesd773ed62007-10-16 23:26:01 -0700428
429static inline int zone_test_and_set_flag(struct zone *zone, zone_flags_t flag)
430{
431 return test_and_set_bit(flag, &zone->flags);
432}
433
David Rientjese815af92007-10-16 23:25:54 -0700434static inline void zone_clear_flag(struct zone *zone, zone_flags_t flag)
435{
436 clear_bit(flag, &zone->flags);
437}
438
439static inline int zone_is_all_unreclaimable(const struct zone *zone)
440{
441 return test_bit(ZONE_ALL_UNRECLAIMABLE, &zone->flags);
442}
David Rientjesd773ed62007-10-16 23:26:01 -0700443
David Rientjese815af92007-10-16 23:25:54 -0700444static inline int zone_is_reclaim_locked(const struct zone *zone)
445{
446 return test_bit(ZONE_RECLAIM_LOCKED, &zone->flags);
447}
David Rientjesd773ed62007-10-16 23:26:01 -0700448
David Rientjes098d7f12007-10-16 23:25:55 -0700449static inline int zone_is_oom_locked(const struct zone *zone)
450{
451 return test_bit(ZONE_OOM_LOCKED, &zone->flags);
452}
David Rientjese815af92007-10-16 23:25:54 -0700453
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454/*
455 * The "priority" of VM scanning is how much of the queues we will scan in one
456 * go. A value of 12 for DEF_PRIORITY implies that we will scan 1/4096th of the
457 * queues ("queue_length >> 12") during an aging round.
458 */
459#define DEF_PRIORITY 12
460
Paul Jackson9276b1bc2006-12-06 20:31:48 -0800461/* Maximum number of zones on a zonelist */
462#define MAX_ZONES_PER_ZONELIST (MAX_NUMNODES * MAX_NR_ZONES)
463
464#ifdef CONFIG_NUMA
Christoph Lameter523b9452007-10-16 01:25:37 -0700465
466/*
467 * The NUMA zonelists are doubled becausse we need zonelists that restrict the
468 * allocations to a single node for GFP_THISNODE.
469 *
Mel Gorman54a6eb52008-04-28 02:12:16 -0700470 * [0] : Zonelist with fallback
471 * [1] : No fallback (GFP_THISNODE)
Christoph Lameter523b9452007-10-16 01:25:37 -0700472 */
Mel Gorman54a6eb52008-04-28 02:12:16 -0700473#define MAX_ZONELISTS 2
Christoph Lameter523b9452007-10-16 01:25:37 -0700474
475
Paul Jackson9276b1bc2006-12-06 20:31:48 -0800476/*
477 * We cache key information from each zonelist for smaller cache
478 * footprint when scanning for free pages in get_page_from_freelist().
479 *
480 * 1) The BITMAP fullzones tracks which zones in a zonelist have come
481 * up short of free memory since the last time (last_fullzone_zap)
482 * we zero'd fullzones.
483 * 2) The array z_to_n[] maps each zone in the zonelist to its node
484 * id, so that we can efficiently evaluate whether that node is
485 * set in the current tasks mems_allowed.
486 *
487 * Both fullzones and z_to_n[] are one-to-one with the zonelist,
488 * indexed by a zones offset in the zonelist zones[] array.
489 *
490 * The get_page_from_freelist() routine does two scans. During the
491 * first scan, we skip zones whose corresponding bit in 'fullzones'
492 * is set or whose corresponding node in current->mems_allowed (which
493 * comes from cpusets) is not set. During the second scan, we bypass
494 * this zonelist_cache, to ensure we look methodically at each zone.
495 *
496 * Once per second, we zero out (zap) fullzones, forcing us to
497 * reconsider nodes that might have regained more free memory.
498 * The field last_full_zap is the time we last zapped fullzones.
499 *
500 * This mechanism reduces the amount of time we waste repeatedly
501 * reexaming zones for free memory when they just came up low on
502 * memory momentarilly ago.
503 *
504 * The zonelist_cache struct members logically belong in struct
505 * zonelist. However, the mempolicy zonelists constructed for
506 * MPOL_BIND are intentionally variable length (and usually much
507 * shorter). A general purpose mechanism for handling structs with
508 * multiple variable length members is more mechanism than we want
509 * here. We resort to some special case hackery instead.
510 *
511 * The MPOL_BIND zonelists don't need this zonelist_cache (in good
512 * part because they are shorter), so we put the fixed length stuff
513 * at the front of the zonelist struct, ending in a variable length
514 * zones[], as is needed by MPOL_BIND.
515 *
516 * Then we put the optional zonelist cache on the end of the zonelist
517 * struct. This optional stuff is found by a 'zlcache_ptr' pointer in
518 * the fixed length portion at the front of the struct. This pointer
519 * both enables us to find the zonelist cache, and in the case of
520 * MPOL_BIND zonelists, (which will just set the zlcache_ptr to NULL)
521 * to know that the zonelist cache is not there.
522 *
523 * The end result is that struct zonelists come in two flavors:
524 * 1) The full, fixed length version, shown below, and
525 * 2) The custom zonelists for MPOL_BIND.
526 * The custom MPOL_BIND zonelists have a NULL zlcache_ptr and no zlcache.
527 *
528 * Even though there may be multiple CPU cores on a node modifying
529 * fullzones or last_full_zap in the same zonelist_cache at the same
530 * time, we don't lock it. This is just hint data - if it is wrong now
531 * and then, the allocator will still function, perhaps a bit slower.
532 */
533
534
535struct zonelist_cache {
Paul Jackson9276b1bc2006-12-06 20:31:48 -0800536 unsigned short z_to_n[MAX_ZONES_PER_ZONELIST]; /* zone->nid */
Paul Jackson7253f4e2006-12-06 20:31:49 -0800537 DECLARE_BITMAP(fullzones, MAX_ZONES_PER_ZONELIST); /* zone full? */
Paul Jackson9276b1bc2006-12-06 20:31:48 -0800538 unsigned long last_full_zap; /* when last zap'd (jiffies) */
539};
540#else
Mel Gorman54a6eb52008-04-28 02:12:16 -0700541#define MAX_ZONELISTS 1
Paul Jackson9276b1bc2006-12-06 20:31:48 -0800542struct zonelist_cache;
543#endif
544
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545/*
Mel Gormandd1a2392008-04-28 02:12:17 -0700546 * This struct contains information about a zone in a zonelist. It is stored
547 * here to avoid dereferences into large structures and lookups of tables
548 */
549struct zoneref {
550 struct zone *zone; /* Pointer to actual zone */
551 int zone_idx; /* zone_idx(zoneref->zone) */
552};
553
554/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700555 * One allocation request operates on a zonelist. A zonelist
556 * is a list of zones, the first one is the 'goal' of the
557 * allocation, the other zones are fallback zones, in decreasing
558 * priority.
559 *
Paul Jackson9276b1bc2006-12-06 20:31:48 -0800560 * If zlcache_ptr is not NULL, then it is just the address of zlcache,
561 * as explained above. If zlcache_ptr is NULL, there is no zlcache.
Mel Gormandd1a2392008-04-28 02:12:17 -0700562 * *
563 * To speed the reading of the zonelist, the zonerefs contain the zone index
564 * of the entry being read. Helper functions to access information given
565 * a struct zoneref are
566 *
567 * zonelist_zone() - Return the struct zone * for an entry in _zonerefs
568 * zonelist_zone_idx() - Return the index of the zone for an entry
569 * zonelist_node_idx() - Return the index of the node for an entry
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570 */
571struct zonelist {
Paul Jackson9276b1bc2006-12-06 20:31:48 -0800572 struct zonelist_cache *zlcache_ptr; // NULL or &zlcache
Mel Gormandd1a2392008-04-28 02:12:17 -0700573 struct zoneref _zonerefs[MAX_ZONES_PER_ZONELIST + 1];
Paul Jackson9276b1bc2006-12-06 20:31:48 -0800574#ifdef CONFIG_NUMA
575 struct zonelist_cache zlcache; // optional ...
576#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577};
578
Mel Gormanc7132162006-09-27 01:49:43 -0700579#ifdef CONFIG_ARCH_POPULATES_NODE_MAP
580struct node_active_region {
581 unsigned long start_pfn;
582 unsigned long end_pfn;
583 int nid;
584};
585#endif /* CONFIG_ARCH_POPULATES_NODE_MAP */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586
Heiko Carstens5b99cd02006-09-27 01:50:01 -0700587#ifndef CONFIG_DISCONTIGMEM
588/* The array of struct pages - for discontigmem use pgdat->lmem_map */
589extern struct page *mem_map;
590#endif
591
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592/*
593 * The pg_data_t structure is used in machines with CONFIG_DISCONTIGMEM
594 * (mostly NUMA machines?) to denote a higher-level memory zone than the
595 * zone denotes.
596 *
597 * On NUMA machines, each NUMA node would have a pg_data_t to describe
598 * it's memory layout.
599 *
600 * Memory statistics and page replacement data structures are maintained on a
601 * per-zone basis.
602 */
603struct bootmem_data;
604typedef struct pglist_data {
605 struct zone node_zones[MAX_NR_ZONES];
Christoph Lameter523b9452007-10-16 01:25:37 -0700606 struct zonelist node_zonelists[MAX_ZONELISTS];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700607 int nr_zones;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700608#ifdef CONFIG_FLAT_NODE_MEM_MAP /* means !SPARSEMEM */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700609 struct page *node_mem_map;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700610#ifdef CONFIG_CGROUP_MEM_RES_CTLR
611 struct page_cgroup *node_page_cgroup;
612#endif
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700613#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700614 struct bootmem_data *bdata;
Dave Hansen208d54e2005-10-29 18:16:52 -0700615#ifdef CONFIG_MEMORY_HOTPLUG
616 /*
617 * Must be held any time you expect node_start_pfn, node_present_pages
618 * or node_spanned_pages stay constant. Holding this will also
619 * guarantee that any pfn_valid() stays that way.
620 *
621 * Nests above zone->lock and zone->size_seqlock.
622 */
623 spinlock_t node_size_lock;
624#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700625 unsigned long node_start_pfn;
626 unsigned long node_present_pages; /* total number of physical pages */
627 unsigned long node_spanned_pages; /* total size of physical page
628 range, including holes */
629 int node_id;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700630 wait_queue_head_t kswapd_wait;
631 struct task_struct *kswapd;
632 int kswapd_max_order;
633} pg_data_t;
634
635#define node_present_pages(nid) (NODE_DATA(nid)->node_present_pages)
636#define node_spanned_pages(nid) (NODE_DATA(nid)->node_spanned_pages)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700637#ifdef CONFIG_FLAT_NODE_MEM_MAP
Dave Hansen408fde82005-06-23 00:07:37 -0700638#define pgdat_page_nr(pgdat, pagenr) ((pgdat)->node_mem_map + (pagenr))
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700639#else
640#define pgdat_page_nr(pgdat, pagenr) pfn_to_page((pgdat)->node_start_pfn + (pagenr))
641#endif
Dave Hansen408fde82005-06-23 00:07:37 -0700642#define nid_page_nr(nid, pagenr) pgdat_page_nr(NODE_DATA(nid),(pagenr))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700643
Dave Hansen208d54e2005-10-29 18:16:52 -0700644#include <linux/memory_hotplug.h>
645
Linus Torvalds1da177e2005-04-16 15:20:36 -0700646void get_zone_counts(unsigned long *active, unsigned long *inactive,
647 unsigned long *free);
648void build_all_zonelists(void);
649void wakeup_kswapd(struct zone *zone, int order);
650int zone_watermark_ok(struct zone *z, int order, unsigned long mark,
Rohit Seth7fb1d9f2005-11-13 16:06:43 -0800651 int classzone_idx, int alloc_flags);
Dave Hansena2f3aa022007-01-10 23:15:30 -0800652enum memmap_context {
653 MEMMAP_EARLY,
654 MEMMAP_HOTPLUG,
655};
Yasunori Goto718127c2006-06-23 02:03:10 -0700656extern int init_currently_empty_zone(struct zone *zone, unsigned long start_pfn,
Dave Hansena2f3aa022007-01-10 23:15:30 -0800657 unsigned long size,
658 enum memmap_context context);
Yasunori Goto718127c2006-06-23 02:03:10 -0700659
Linus Torvalds1da177e2005-04-16 15:20:36 -0700660#ifdef CONFIG_HAVE_MEMORY_PRESENT
661void memory_present(int nid, unsigned long start, unsigned long end);
662#else
663static inline void memory_present(int nid, unsigned long start, unsigned long end) {}
664#endif
665
666#ifdef CONFIG_NEED_NODE_MEMMAP_SIZE
667unsigned long __init node_memmap_size_bytes(int, unsigned long, unsigned long);
668#endif
669
670/*
671 * zone_idx() returns 0 for the ZONE_DMA zone, 1 for the ZONE_NORMAL zone, etc.
672 */
673#define zone_idx(zone) ((zone) - (zone)->zone_pgdat->node_zones)
674
Con Kolivasf3fe6512006-01-06 00:11:15 -0800675static inline int populated_zone(struct zone *zone)
676{
677 return (!!zone->present_pages);
678}
679
Mel Gorman2a1e2742007-07-17 04:03:12 -0700680extern int movable_zone;
681
682static inline int zone_movable_is_highmem(void)
683{
684#if defined(CONFIG_HIGHMEM) && defined(CONFIG_ARCH_POPULATES_NODE_MAP)
685 return movable_zone == ZONE_HIGHMEM;
686#else
687 return 0;
688#endif
689}
690
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700691static inline int is_highmem_idx(enum zone_type idx)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700692{
Christoph Lametere53ef382006-09-25 23:31:14 -0700693#ifdef CONFIG_HIGHMEM
Mel Gorman2a1e2742007-07-17 04:03:12 -0700694 return (idx == ZONE_HIGHMEM ||
695 (idx == ZONE_MOVABLE && zone_movable_is_highmem()));
Christoph Lametere53ef382006-09-25 23:31:14 -0700696#else
697 return 0;
698#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700699}
700
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700701static inline int is_normal_idx(enum zone_type idx)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702{
703 return (idx == ZONE_NORMAL);
704}
Nick Piggin9328b8f2006-01-06 00:11:10 -0800705
Linus Torvalds1da177e2005-04-16 15:20:36 -0700706/**
707 * is_highmem - helper function to quickly check if a struct zone is a
708 * highmem zone or not. This is an attempt to keep references
709 * to ZONE_{DMA/NORMAL/HIGHMEM/etc} in general code to a minimum.
710 * @zone - pointer to struct zone variable
711 */
712static inline int is_highmem(struct zone *zone)
713{
Christoph Lametere53ef382006-09-25 23:31:14 -0700714#ifdef CONFIG_HIGHMEM
Harvey Harrisonddc81ed2008-04-28 02:12:07 -0700715 int zone_off = (char *)zone - (char *)zone->zone_pgdat->node_zones;
716 return zone_off == ZONE_HIGHMEM * sizeof(*zone) ||
717 (zone_off == ZONE_MOVABLE * sizeof(*zone) &&
718 zone_movable_is_highmem());
Christoph Lametere53ef382006-09-25 23:31:14 -0700719#else
720 return 0;
721#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722}
723
724static inline int is_normal(struct zone *zone)
725{
726 return zone == zone->zone_pgdat->node_zones + ZONE_NORMAL;
727}
728
Nick Piggin9328b8f2006-01-06 00:11:10 -0800729static inline int is_dma32(struct zone *zone)
730{
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700731#ifdef CONFIG_ZONE_DMA32
Nick Piggin9328b8f2006-01-06 00:11:10 -0800732 return zone == zone->zone_pgdat->node_zones + ZONE_DMA32;
Christoph Lameterfb0e7942006-09-25 23:31:13 -0700733#else
734 return 0;
735#endif
Nick Piggin9328b8f2006-01-06 00:11:10 -0800736}
737
738static inline int is_dma(struct zone *zone)
739{
Christoph Lameter4b51d662007-02-10 01:43:10 -0800740#ifdef CONFIG_ZONE_DMA
Nick Piggin9328b8f2006-01-06 00:11:10 -0800741 return zone == zone->zone_pgdat->node_zones + ZONE_DMA;
Christoph Lameter4b51d662007-02-10 01:43:10 -0800742#else
743 return 0;
744#endif
Nick Piggin9328b8f2006-01-06 00:11:10 -0800745}
746
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747/* These two functions are used to setup the per zone pages min values */
748struct ctl_table;
749struct file;
750int min_free_kbytes_sysctl_handler(struct ctl_table *, int, struct file *,
751 void __user *, size_t *, loff_t *);
752extern int sysctl_lowmem_reserve_ratio[MAX_NR_ZONES-1];
753int lowmem_reserve_ratio_sysctl_handler(struct ctl_table *, int, struct file *,
754 void __user *, size_t *, loff_t *);
Rohit Seth8ad4b1f2006-01-08 01:00:40 -0800755int percpu_pagelist_fraction_sysctl_handler(struct ctl_table *, int, struct file *,
756 void __user *, size_t *, loff_t *);
Christoph Lameter96146342006-07-03 00:24:13 -0700757int sysctl_min_unmapped_ratio_sysctl_handler(struct ctl_table *, int,
758 struct file *, void __user *, size_t *, loff_t *);
Christoph Lameter0ff38492006-09-25 23:31:52 -0700759int sysctl_min_slab_ratio_sysctl_handler(struct ctl_table *, int,
760 struct file *, void __user *, size_t *, loff_t *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761
KAMEZAWA Hiroyukif0c0b2b2007-07-15 23:38:01 -0700762extern int numa_zonelist_order_handler(struct ctl_table *, int,
763 struct file *, void __user *, size_t *, loff_t *);
764extern char numa_zonelist_order[];
765#define NUMA_ZONELIST_ORDER_LEN 16 /* string buffer size */
766
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767#include <linux/topology.h>
768/* Returns the number of the current Node. */
Andi Kleen69d81fc2005-11-05 17:25:53 +0100769#ifndef numa_node_id
Ingo Molnar39c715b2005-06-21 17:14:34 -0700770#define numa_node_id() (cpu_to_node(raw_smp_processor_id()))
Andi Kleen69d81fc2005-11-05 17:25:53 +0100771#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772
Dave Hansen93b75042005-06-23 00:07:47 -0700773#ifndef CONFIG_NEED_MULTIPLE_NODES
Linus Torvalds1da177e2005-04-16 15:20:36 -0700774
775extern struct pglist_data contig_page_data;
776#define NODE_DATA(nid) (&contig_page_data)
777#define NODE_MEM_MAP(nid) mem_map
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778
Dave Hansen93b75042005-06-23 00:07:47 -0700779#else /* CONFIG_NEED_MULTIPLE_NODES */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700780
781#include <asm/mmzone.h>
782
Dave Hansen93b75042005-06-23 00:07:47 -0700783#endif /* !CONFIG_NEED_MULTIPLE_NODES */
Dave Hansen348f8b62005-06-23 00:07:40 -0700784
KAMEZAWA Hiroyuki95144c72006-03-27 01:16:02 -0800785extern struct pglist_data *first_online_pgdat(void);
786extern struct pglist_data *next_online_pgdat(struct pglist_data *pgdat);
787extern struct zone *next_zone(struct zone *zone);
KAMEZAWA Hiroyuki8357f862006-03-27 01:15:57 -0800788
789/**
Fernando Luis Vazquez Cao12d15f02008-05-23 13:05:01 -0700790 * for_each_online_pgdat - helper macro to iterate over all online nodes
KAMEZAWA Hiroyuki8357f862006-03-27 01:15:57 -0800791 * @pgdat - pointer to a pg_data_t variable
792 */
793#define for_each_online_pgdat(pgdat) \
794 for (pgdat = first_online_pgdat(); \
795 pgdat; \
796 pgdat = next_online_pgdat(pgdat))
KAMEZAWA Hiroyuki8357f862006-03-27 01:15:57 -0800797/**
798 * for_each_zone - helper macro to iterate over all memory zones
799 * @zone - pointer to struct zone variable
800 *
801 * The user only needs to declare the zone variable, for_each_zone
802 * fills it in.
803 */
804#define for_each_zone(zone) \
805 for (zone = (first_online_pgdat())->node_zones; \
806 zone; \
807 zone = next_zone(zone))
808
Mel Gormandd1a2392008-04-28 02:12:17 -0700809static inline struct zone *zonelist_zone(struct zoneref *zoneref)
810{
811 return zoneref->zone;
812}
813
814static inline int zonelist_zone_idx(struct zoneref *zoneref)
815{
816 return zoneref->zone_idx;
817}
818
819static inline int zonelist_node_idx(struct zoneref *zoneref)
820{
821#ifdef CONFIG_NUMA
822 /* zone_to_nid not available in this context */
823 return zoneref->zone->node;
824#else
825 return 0;
826#endif /* CONFIG_NUMA */
827}
828
Mel Gorman19770b32008-04-28 02:12:18 -0700829/**
830 * next_zones_zonelist - Returns the next zone at or below highest_zoneidx within the allowed nodemask using a cursor within a zonelist as a starting point
831 * @z - The cursor used as a starting point for the search
832 * @highest_zoneidx - The zone index of the highest zone to return
833 * @nodes - An optional nodemask to filter the zonelist with
834 * @zone - The first suitable zone found is returned via this parameter
835 *
836 * This function returns the next zone at or below a given zone index that is
837 * within the allowed nodemask using a cursor as the starting point for the
Mel Gorman5bead2a2008-09-13 02:33:19 -0700838 * search. The zoneref returned is a cursor that represents the current zone
839 * being examined. It should be advanced by one before calling
840 * next_zones_zonelist again.
Mel Gorman19770b32008-04-28 02:12:18 -0700841 */
842struct zoneref *next_zones_zonelist(struct zoneref *z,
843 enum zone_type highest_zoneidx,
844 nodemask_t *nodes,
845 struct zone **zone);
Mel Gormandd1a2392008-04-28 02:12:17 -0700846
Mel Gorman19770b32008-04-28 02:12:18 -0700847/**
848 * first_zones_zonelist - Returns the first zone at or below highest_zoneidx within the allowed nodemask in a zonelist
849 * @zonelist - The zonelist to search for a suitable zone
850 * @highest_zoneidx - The zone index of the highest zone to return
851 * @nodes - An optional nodemask to filter the zonelist with
852 * @zone - The first suitable zone found is returned via this parameter
853 *
854 * This function returns the first zone at or below a given zone index that is
855 * within the allowed nodemask. The zoneref returned is a cursor that can be
Mel Gorman5bead2a2008-09-13 02:33:19 -0700856 * used to iterate the zonelist with next_zones_zonelist by advancing it by
857 * one before calling.
Mel Gorman19770b32008-04-28 02:12:18 -0700858 */
Mel Gormandd1a2392008-04-28 02:12:17 -0700859static inline struct zoneref *first_zones_zonelist(struct zonelist *zonelist,
Mel Gorman19770b32008-04-28 02:12:18 -0700860 enum zone_type highest_zoneidx,
861 nodemask_t *nodes,
862 struct zone **zone)
Mel Gorman54a6eb52008-04-28 02:12:16 -0700863{
Mel Gorman19770b32008-04-28 02:12:18 -0700864 return next_zones_zonelist(zonelist->_zonerefs, highest_zoneidx, nodes,
865 zone);
Mel Gorman54a6eb52008-04-28 02:12:16 -0700866}
867
Mel Gorman19770b32008-04-28 02:12:18 -0700868/**
869 * for_each_zone_zonelist_nodemask - helper macro to iterate over valid zones in a zonelist at or below a given zone index and within a nodemask
870 * @zone - The current zone in the iterator
871 * @z - The current pointer within zonelist->zones being iterated
872 * @zlist - The zonelist being iterated
873 * @highidx - The zone index of the highest zone to return
874 * @nodemask - Nodemask allowed by the allocator
875 *
876 * This iterator iterates though all zones at or below a given zone index and
877 * within a given nodemask
878 */
879#define for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, nodemask) \
880 for (z = first_zones_zonelist(zlist, highidx, nodemask, &zone); \
881 zone; \
Mel Gorman5bead2a2008-09-13 02:33:19 -0700882 z = next_zones_zonelist(++z, highidx, nodemask, &zone)) \
Mel Gorman54a6eb52008-04-28 02:12:16 -0700883
884/**
885 * for_each_zone_zonelist - helper macro to iterate over valid zones in a zonelist at or below a given zone index
886 * @zone - The current zone in the iterator
887 * @z - The current pointer within zonelist->zones being iterated
888 * @zlist - The zonelist being iterated
889 * @highidx - The zone index of the highest zone to return
890 *
891 * This iterator iterates though all zones at or below a given zone index.
892 */
893#define for_each_zone_zonelist(zone, z, zlist, highidx) \
Mel Gorman19770b32008-04-28 02:12:18 -0700894 for_each_zone_zonelist_nodemask(zone, z, zlist, highidx, NULL)
Mel Gorman54a6eb52008-04-28 02:12:16 -0700895
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700896#ifdef CONFIG_SPARSEMEM
897#include <asm/sparsemem.h>
898#endif
899
Mel Gormanc7132162006-09-27 01:49:43 -0700900#if !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID) && \
901 !defined(CONFIG_ARCH_POPULATES_NODE_MAP)
Andrew Mortonb4544562008-04-28 02:12:39 -0700902static inline unsigned long early_pfn_to_nid(unsigned long pfn)
903{
904 return 0;
905}
Andy Whitcroftb159d432005-06-23 00:07:52 -0700906#endif
907
Andy Whitcroft2bdaf112006-01-06 00:10:53 -0800908#ifdef CONFIG_FLATMEM
909#define pfn_to_nid(pfn) (0)
910#endif
911
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700912#define pfn_to_section_nr(pfn) ((pfn) >> PFN_SECTION_SHIFT)
913#define section_nr_to_pfn(sec) ((sec) << PFN_SECTION_SHIFT)
914
915#ifdef CONFIG_SPARSEMEM
916
917/*
918 * SECTION_SHIFT #bits space required to store a section #
919 *
920 * PA_SECTION_SHIFT physical address to/from section number
921 * PFN_SECTION_SHIFT pfn to/from section number
922 */
923#define SECTIONS_SHIFT (MAX_PHYSMEM_BITS - SECTION_SIZE_BITS)
924
925#define PA_SECTION_SHIFT (SECTION_SIZE_BITS)
926#define PFN_SECTION_SHIFT (SECTION_SIZE_BITS - PAGE_SHIFT)
927
928#define NR_MEM_SECTIONS (1UL << SECTIONS_SHIFT)
929
930#define PAGES_PER_SECTION (1UL << PFN_SECTION_SHIFT)
931#define PAGE_SECTION_MASK (~(PAGES_PER_SECTION-1))
932
Mel Gorman835c1342007-10-16 01:25:47 -0700933#define SECTION_BLOCKFLAGS_BITS \
Mel Gormand9c23402007-10-16 01:26:01 -0700934 ((1UL << (PFN_SECTION_SHIFT - pageblock_order)) * NR_PAGEBLOCK_BITS)
Mel Gorman835c1342007-10-16 01:25:47 -0700935
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700936#if (MAX_ORDER - 1 + PAGE_SHIFT) > SECTION_SIZE_BITS
937#error Allocator MAX_ORDER exceeds SECTION_SIZE
938#endif
939
940struct page;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700941struct page_cgroup;
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700942struct mem_section {
Andy Whitcroft29751f62005-06-23 00:08:00 -0700943 /*
944 * This is, logically, a pointer to an array of struct
945 * pages. However, it is stored with some other magic.
946 * (see sparse.c::sparse_init_one_section())
947 *
Andy Whitcroft30c253e2006-06-23 02:03:41 -0700948 * Additionally during early boot we encode node id of
949 * the location of the section here to guide allocation.
950 * (see sparse.c::memory_present())
951 *
Andy Whitcroft29751f62005-06-23 00:08:00 -0700952 * Making it a UL at least makes someone do a cast
953 * before using it wrong.
954 */
955 unsigned long section_mem_map;
Mel Gorman5c0e3062007-10-16 01:25:56 -0700956
957 /* See declaration of similar field in struct zone */
958 unsigned long *pageblock_flags;
KAMEZAWA Hiroyuki52d4b9a2008-10-18 20:28:16 -0700959#ifdef CONFIG_CGROUP_MEM_RES_CTLR
960 /*
961 * If !SPARSEMEM, pgdat doesn't have page_cgroup pointer. We use
962 * section. (see memcontrol.h/page_cgroup.h about this.)
963 */
964 struct page_cgroup *page_cgroup;
965 unsigned long pad;
966#endif
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700967};
968
Bob Picco3e347262005-09-03 15:54:28 -0700969#ifdef CONFIG_SPARSEMEM_EXTREME
970#define SECTIONS_PER_ROOT (PAGE_SIZE / sizeof (struct mem_section))
Bob Picco802f1922005-09-03 15:54:26 -0700971#else
Bob Picco3e347262005-09-03 15:54:28 -0700972#define SECTIONS_PER_ROOT 1
973#endif
Bob Picco802f1922005-09-03 15:54:26 -0700974
Bob Picco3e347262005-09-03 15:54:28 -0700975#define SECTION_NR_TO_ROOT(sec) ((sec) / SECTIONS_PER_ROOT)
976#define NR_SECTION_ROOTS (NR_MEM_SECTIONS / SECTIONS_PER_ROOT)
977#define SECTION_ROOT_MASK (SECTIONS_PER_ROOT - 1)
978
979#ifdef CONFIG_SPARSEMEM_EXTREME
980extern struct mem_section *mem_section[NR_SECTION_ROOTS];
981#else
982extern struct mem_section mem_section[NR_SECTION_ROOTS][SECTIONS_PER_ROOT];
983#endif
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700984
Andy Whitcroft29751f62005-06-23 00:08:00 -0700985static inline struct mem_section *__nr_to_section(unsigned long nr)
986{
Bob Picco3e347262005-09-03 15:54:28 -0700987 if (!mem_section[SECTION_NR_TO_ROOT(nr)])
988 return NULL;
989 return &mem_section[SECTION_NR_TO_ROOT(nr)][nr & SECTION_ROOT_MASK];
Andy Whitcroft29751f62005-06-23 00:08:00 -0700990}
Dave Hansen4ca644d2005-10-29 18:16:51 -0700991extern int __section_nr(struct mem_section* ms);
Yasunori Goto04753272008-04-28 02:13:31 -0700992extern unsigned long usemap_size(void);
Andy Whitcroft29751f62005-06-23 00:08:00 -0700993
994/*
995 * We use the lower bits of the mem_map pointer to store
996 * a little bit of information. There should be at least
997 * 3 bits here due to 32-bit alignment.
998 */
999#define SECTION_MARKED_PRESENT (1UL<<0)
1000#define SECTION_HAS_MEM_MAP (1UL<<1)
1001#define SECTION_MAP_LAST_BIT (1UL<<2)
1002#define SECTION_MAP_MASK (~(SECTION_MAP_LAST_BIT-1))
Andy Whitcroft30c253e2006-06-23 02:03:41 -07001003#define SECTION_NID_SHIFT 2
Andy Whitcroft29751f62005-06-23 00:08:00 -07001004
1005static inline struct page *__section_mem_map_addr(struct mem_section *section)
1006{
1007 unsigned long map = section->section_mem_map;
1008 map &= SECTION_MAP_MASK;
1009 return (struct page *)map;
1010}
1011
Andy Whitcroft540557b2007-10-16 01:24:11 -07001012static inline int present_section(struct mem_section *section)
Andy Whitcroft29751f62005-06-23 00:08:00 -07001013{
Bob Picco802f1922005-09-03 15:54:26 -07001014 return (section && (section->section_mem_map & SECTION_MARKED_PRESENT));
Andy Whitcroft29751f62005-06-23 00:08:00 -07001015}
1016
Andy Whitcroft540557b2007-10-16 01:24:11 -07001017static inline int present_section_nr(unsigned long nr)
1018{
1019 return present_section(__nr_to_section(nr));
1020}
1021
1022static inline int valid_section(struct mem_section *section)
Andy Whitcroft29751f62005-06-23 00:08:00 -07001023{
Bob Picco802f1922005-09-03 15:54:26 -07001024 return (section && (section->section_mem_map & SECTION_HAS_MEM_MAP));
Andy Whitcroft29751f62005-06-23 00:08:00 -07001025}
1026
1027static inline int valid_section_nr(unsigned long nr)
1028{
1029 return valid_section(__nr_to_section(nr));
1030}
1031
Andy Whitcroftd41dee32005-06-23 00:07:54 -07001032static inline struct mem_section *__pfn_to_section(unsigned long pfn)
1033{
Andy Whitcroft29751f62005-06-23 00:08:00 -07001034 return __nr_to_section(pfn_to_section_nr(pfn));
Andy Whitcroftd41dee32005-06-23 00:07:54 -07001035}
1036
Andy Whitcroftd41dee32005-06-23 00:07:54 -07001037static inline int pfn_valid(unsigned long pfn)
1038{
1039 if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
1040 return 0;
Andy Whitcroft29751f62005-06-23 00:08:00 -07001041 return valid_section(__nr_to_section(pfn_to_section_nr(pfn)));
Andy Whitcroftd41dee32005-06-23 00:07:54 -07001042}
1043
Andy Whitcroft540557b2007-10-16 01:24:11 -07001044static inline int pfn_present(unsigned long pfn)
1045{
1046 if (pfn_to_section_nr(pfn) >= NR_MEM_SECTIONS)
1047 return 0;
1048 return present_section(__nr_to_section(pfn_to_section_nr(pfn)));
1049}
1050
Andy Whitcroftd41dee32005-06-23 00:07:54 -07001051/*
1052 * These are _only_ used during initialisation, therefore they
1053 * can use __initdata ... They could have names to indicate
1054 * this restriction.
1055 */
1056#ifdef CONFIG_NUMA
Andy Whitcroft161599f2006-01-06 00:10:54 -08001057#define pfn_to_nid(pfn) \
1058({ \
1059 unsigned long __pfn_to_nid_pfn = (pfn); \
1060 page_to_nid(pfn_to_page(__pfn_to_nid_pfn)); \
1061})
Andy Whitcroft2bdaf112006-01-06 00:10:53 -08001062#else
1063#define pfn_to_nid(pfn) (0)
Andy Whitcroftd41dee32005-06-23 00:07:54 -07001064#endif
1065
Andy Whitcroftd41dee32005-06-23 00:07:54 -07001066#define early_pfn_valid(pfn) pfn_valid(pfn)
1067void sparse_init(void);
1068#else
1069#define sparse_init() do {} while (0)
Dave Hansen28ae55c2005-09-03 15:54:29 -07001070#define sparse_index_init(_sec, _nid) do {} while (0)
Andy Whitcroftd41dee32005-06-23 00:07:54 -07001071#endif /* CONFIG_SPARSEMEM */
1072
Andy Whitcroft75167952006-10-21 10:24:14 -07001073#ifdef CONFIG_NODES_SPAN_OTHER_NODES
1074#define early_pfn_in_nid(pfn, nid) (early_pfn_to_nid(pfn) == (nid))
1075#else
1076#define early_pfn_in_nid(pfn, nid) (1)
1077#endif
1078
Andy Whitcroftd41dee32005-06-23 00:07:54 -07001079#ifndef early_pfn_valid
1080#define early_pfn_valid(pfn) (1)
1081#endif
1082
1083void memory_present(int nid, unsigned long start, unsigned long end);
1084unsigned long __init node_memmap_size_bytes(int, unsigned long, unsigned long);
1085
Andy Whitcroft14e07292007-05-06 14:49:14 -07001086/*
1087 * If it is possible to have holes within a MAX_ORDER_NR_PAGES, then we
1088 * need to check pfn validility within that MAX_ORDER_NR_PAGES block.
1089 * pfn_valid_within() should be used in this case; we optimise this away
1090 * when we have no holes within a MAX_ORDER_NR_PAGES block.
1091 */
1092#ifdef CONFIG_HOLES_IN_ZONE
1093#define pfn_valid_within(pfn) pfn_valid(pfn)
1094#else
1095#define pfn_valid_within(pfn) (1)
1096#endif
1097
Christoph Lameter97965472008-04-28 02:12:54 -07001098#endif /* !__GENERATING_BOUNDS.H */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001099#endif /* !__ASSEMBLY__ */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001100#endif /* _LINUX_MMZONE_H */