blob: 4390ac6c106e6124d653f18948ec296173527521 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/* internal.h: mm/ internal definitions
2 *
3 * Copyright (C) 2004 Red Hat, Inc. All Rights Reserved.
4 * Written by David Howells (dhowells@redhat.com)
5 *
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version
9 * 2 of the License, or (at your option) any later version.
10 */
Nick Piggin0f8053a2006-03-22 00:08:33 -080011#ifndef __MM_INTERNAL_H
12#define __MM_INTERNAL_H
13
14#include <linux/mm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070015
Jan Beulich42b77722008-07-23 21:27:10 -070016void free_pgtables(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
17 unsigned long floor, unsigned long ceiling);
18
Nick Piggin7835e982006-03-22 00:08:40 -080019static inline void set_page_count(struct page *page, int v)
Nick Piggin77a8a782006-01-06 00:10:57 -080020{
Nick Piggin7835e982006-03-22 00:08:40 -080021 atomic_set(&page->_count, v);
22}
23
24/*
25 * Turn a non-refcounted page (->_count == 0) into refcounted with
26 * a count of one.
27 */
28static inline void set_page_refcounted(struct page *page)
29{
Qi Yongae1276b2008-02-04 22:29:27 -080030 VM_BUG_ON(PageTail(page));
Nick Piggin725d7042006-09-25 23:30:55 -070031 VM_BUG_ON(atomic_read(&page->_count));
Nick Piggin77a8a782006-01-06 00:10:57 -080032 set_page_count(page, 1);
Nick Piggin77a8a782006-01-06 00:10:57 -080033}
34
Andrea Arcangeli70b50f92011-11-02 13:36:59 -070035static inline void __get_page_tail_foll(struct page *page,
36 bool get_page_head)
37{
38 /*
39 * If we're getting a tail page, the elevated page->_count is
40 * required only in the head page and we will elevate the head
41 * page->_count and tail page->_mapcount.
42 *
43 * We elevate page_tail->_mapcount for tail pages to force
44 * page_tail->_count to be zero at all times to avoid getting
45 * false positives from get_page_unless_zero() with
46 * speculative page access (like in
47 * page_cache_get_speculative()) on tail pages.
48 */
49 VM_BUG_ON(atomic_read(&page->first_page->_count) <= 0);
50 VM_BUG_ON(atomic_read(&page->_count) != 0);
51 VM_BUG_ON(page_mapcount(page) < 0);
52 if (get_page_head)
53 atomic_inc(&page->first_page->_count);
54 atomic_inc(&page->_mapcount);
55}
56
57/*
58 * This is meant to be called as the FOLL_GET operation of
59 * follow_page() and it must be called while holding the proper PT
60 * lock while the pte (or pmd_trans_huge) is still mapping the page.
61 */
62static inline void get_page_foll(struct page *page)
63{
64 if (unlikely(PageTail(page)))
65 /*
66 * This is safe only because
67 * __split_huge_page_refcount() can't run under
68 * get_page_foll() because we hold the proper PT lock.
69 */
70 __get_page_tail_foll(page, true);
71 else {
72 /*
73 * Getting a normal page or the head of a compound page
74 * requires to already have an elevated page->_count.
75 */
76 VM_BUG_ON(atomic_read(&page->_count) <= 0);
77 atomic_inc(&page->_count);
78 }
79}
80
Hugh Dickins03f64622009-09-21 17:03:35 -070081extern unsigned long highest_memmap_pfn;
82
Lee Schermerhorn894bc312008-10-18 20:26:39 -070083/*
84 * in mm/vmscan.c:
85 */
Nick Piggin62695a82008-10-18 20:26:09 -070086extern int isolate_lru_page(struct page *page);
Lee Schermerhorn894bc312008-10-18 20:26:39 -070087extern void putback_lru_page(struct page *page);
Nick Piggin62695a82008-10-18 20:26:09 -070088
Lee Schermerhorn894bc312008-10-18 20:26:39 -070089/*
Bob Liu62190492012-12-11 16:00:37 -080090 * in mm/rmap.c:
91 */
92extern pmd_t *mm_find_pmd(struct mm_struct *mm, unsigned long address);
93
94/*
Lee Schermerhorn894bc312008-10-18 20:26:39 -070095 * in mm/page_alloc.c
96 */
Yasunori Goto0c0a4a52008-04-28 02:13:34 -070097extern void __free_pages_bootmem(struct page *page, unsigned int order);
Wu Fengguang20a03072009-06-16 15:32:22 -070098extern void prep_compound_page(struct page *page, unsigned long order);
Wu Fengguang8d22ba12009-12-16 12:19:58 +010099#ifdef CONFIG_MEMORY_FAILURE
100extern bool is_free_buddy_page(struct page *page);
101#endif
Wu Fengguang20a03072009-06-16 15:32:22 -0700102
Michal Nazarewiczff9543f2011-12-29 13:09:50 +0100103#if defined CONFIG_COMPACTION || defined CONFIG_CMA
104
105/*
106 * in mm/compaction.c
107 */
108/*
109 * compact_control is used to track pages being migrated and the free pages
110 * they are being migrated to during memory compaction. The free_pfn starts
111 * at the end of a zone and migrate_pfn begins at the start. Movable pages
112 * are moved to the end of a zone during a compaction run and the run
113 * completes when free_pfn <= migrate_pfn
114 */
115struct compact_control {
116 struct list_head freepages; /* List of free pages to migrate to */
117 struct list_head migratepages; /* List of pages being migrated */
118 unsigned long nr_freepages; /* Number of isolated free pages */
119 unsigned long nr_migratepages; /* Number of pages to migrate */
120 unsigned long free_pfn; /* isolate_freepages search base */
121 unsigned long migrate_pfn; /* isolate_migratepages search base */
Linus Torvalds68e3e922012-06-03 20:05:57 -0700122 bool sync; /* Synchronous migration */
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700123 bool ignore_skip_hint; /* Scan blocks even if marked skip */
Mel Gormanc89511a2012-10-08 16:32:45 -0700124 bool finished_update_free; /* True when the zone cached pfns are
125 * no longer being updated
126 */
127 bool finished_update_migrate;
Michal Nazarewiczff9543f2011-12-29 13:09:50 +0100128
129 int order; /* order a direct compactor needs */
130 int migratetype; /* MOVABLE, RECLAIMABLE etc */
131 struct zone *zone;
Shaohua Lie64c5232012-10-08 16:32:27 -0700132 bool contended; /* True if a lock was contended */
Michal Nazarewiczff9543f2011-12-29 13:09:50 +0100133};
134
135unsigned long
Mel Gormanbb13ffe2012-10-08 16:32:41 -0700136isolate_freepages_range(struct compact_control *cc,
137 unsigned long start_pfn, unsigned long end_pfn);
Michal Nazarewiczff9543f2011-12-29 13:09:50 +0100138unsigned long
139isolate_migratepages_range(struct zone *zone, struct compact_control *cc,
Minchan Kime46a2872012-10-08 16:33:48 -0700140 unsigned long low_pfn, unsigned long end_pfn, bool unevictable);
Michal Nazarewiczff9543f2011-12-29 13:09:50 +0100141
142#endif
Nick Piggin0f8053a2006-03-22 00:08:33 -0800143
Mel Gorman48f13bf2007-10-16 01:26:10 -0700144/*
145 * function for dealing with page's order in buddy system.
146 * zone->lock is already acquired when we use these.
147 * So, we don't need atomic page->flags operations here.
148 */
149static inline unsigned long page_order(struct page *page)
150{
KAMEZAWA Hiroyuki572438f2010-10-26 14:22:08 -0700151 /* PageBuddy() must be checked by the caller */
Mel Gorman48f13bf2007-10-16 01:26:10 -0700152 return page_private(page);
153}
Alexander van Heukelumb5a0e012008-02-23 15:24:06 -0800154
Namhyung Kim6038def2011-05-24 17:11:22 -0700155/* mm/util.c */
156void __vma_link_list(struct mm_struct *mm, struct vm_area_struct *vma,
157 struct vm_area_struct *prev, struct rb_node *rb_parent);
158
Hugh Dickinsaf8e3352009-12-14 17:58:59 -0800159#ifdef CONFIG_MMU
Michel Lespinassecea10a12013-02-22 16:32:44 -0800160extern long __mlock_vma_pages_range(struct vm_area_struct *vma,
161 unsigned long start, unsigned long end, int *nonblocking);
Hugh Dickinsaf8e3352009-12-14 17:58:59 -0800162extern void munlock_vma_pages_range(struct vm_area_struct *vma,
163 unsigned long start, unsigned long end);
164static inline void munlock_vma_pages_all(struct vm_area_struct *vma)
165{
166 munlock_vma_pages_range(vma, vma->vm_start, vma->vm_end);
167}
168
Nick Pigginb291f002008-10-18 20:26:44 -0700169/*
Hugh Dickins39b5f292012-10-08 16:33:18 -0700170 * Called only in fault path, to determine if a new page is being
171 * mapped into a LOCKED vma. If it is, mark page as mlocked.
Nick Pigginb291f002008-10-18 20:26:44 -0700172 */
Ying Han096a7cf2012-05-29 15:06:25 -0700173static inline int mlocked_vma_newpage(struct vm_area_struct *vma,
174 struct page *page)
Nick Pigginb291f002008-10-18 20:26:44 -0700175{
176 VM_BUG_ON(PageLRU(page));
177
178 if (likely((vma->vm_flags & (VM_LOCKED | VM_SPECIAL)) != VM_LOCKED))
179 return 0;
180
Nick Piggin5344b7e2008-10-18 20:26:51 -0700181 if (!TestSetPageMlocked(page)) {
David Rientjes8449d212012-10-08 16:34:06 -0700182 mod_zone_page_state(page_zone(page), NR_MLOCK,
183 hpage_nr_pages(page));
Nick Piggin5344b7e2008-10-18 20:26:51 -0700184 count_vm_event(UNEVICTABLE_PGMLOCKED);
185 }
Nick Pigginb291f002008-10-18 20:26:44 -0700186 return 1;
187}
188
189/*
Hugh Dickins73848b42009-12-14 17:59:22 -0800190 * must be called with vma's mmap_sem held for read or write, and page locked.
Nick Pigginb291f002008-10-18 20:26:44 -0700191 */
192extern void mlock_vma_page(struct page *page);
Michel Lespinasseff6a6da2013-02-27 17:02:44 -0800193extern unsigned int munlock_vma_page(struct page *page);
Nick Pigginb291f002008-10-18 20:26:44 -0700194
195/*
196 * Clear the page's PageMlocked(). This can be useful in a situation where
197 * we want to unconditionally remove a page from the pagecache -- e.g.,
198 * on truncation or freeing.
199 *
200 * It is legal to call this function for any page, mlocked or not.
201 * If called for a page that is still mapped by mlocked vmas, all we do
202 * is revert to lazy LRU behaviour -- semantics are not broken.
203 */
Hugh Dickinse6c509f2012-10-08 16:33:19 -0700204extern void clear_page_mlock(struct page *page);
Nick Pigginb291f002008-10-18 20:26:44 -0700205
206/*
207 * mlock_migrate_page - called only from migrate_page_copy() to
Nick Piggin5344b7e2008-10-18 20:26:51 -0700208 * migrate the Mlocked page flag; update statistics.
Nick Pigginb291f002008-10-18 20:26:44 -0700209 */
210static inline void mlock_migrate_page(struct page *newpage, struct page *page)
211{
Nick Piggin5344b7e2008-10-18 20:26:51 -0700212 if (TestClearPageMlocked(page)) {
213 unsigned long flags;
Mel Gormanb32967f2012-11-19 12:35:47 +0000214 int nr_pages = hpage_nr_pages(page);
Nick Piggin5344b7e2008-10-18 20:26:51 -0700215
216 local_irq_save(flags);
Mel Gormanb32967f2012-11-19 12:35:47 +0000217 __mod_zone_page_state(page_zone(page), NR_MLOCK, -nr_pages);
Nick Pigginb291f002008-10-18 20:26:44 -0700218 SetPageMlocked(newpage);
Mel Gormanb32967f2012-11-19 12:35:47 +0000219 __mod_zone_page_state(page_zone(newpage), NR_MLOCK, nr_pages);
Nick Piggin5344b7e2008-10-18 20:26:51 -0700220 local_irq_restore(flags);
221 }
Nick Pigginb291f002008-10-18 20:26:44 -0700222}
223
Mel Gormanb32967f2012-11-19 12:35:47 +0000224extern pmd_t maybe_pmd_mkwrite(pmd_t pmd, struct vm_area_struct *vma);
225
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800226#ifdef CONFIG_TRANSPARENT_HUGEPAGE
227extern unsigned long vma_address(struct page *page,
228 struct vm_area_struct *vma);
229#endif
Hugh Dickinsaf8e3352009-12-14 17:58:59 -0800230#else /* !CONFIG_MMU */
Ying Han096a7cf2012-05-29 15:06:25 -0700231static inline int mlocked_vma_newpage(struct vm_area_struct *v, struct page *p)
Nick Pigginb291f002008-10-18 20:26:44 -0700232{
233 return 0;
234}
235static inline void clear_page_mlock(struct page *page) { }
236static inline void mlock_vma_page(struct page *page) { }
237static inline void mlock_migrate_page(struct page *new, struct page *old) { }
238
Hugh Dickinsaf8e3352009-12-14 17:58:59 -0800239#endif /* !CONFIG_MMU */
Lee Schermerhorn894bc312008-10-18 20:26:39 -0700240
Alexander van Heukelumb5a0e012008-02-23 15:24:06 -0800241/*
Andy Whitcroft69d177c2008-11-06 12:53:26 -0800242 * Return the mem_map entry representing the 'offset' subpage within
243 * the maximally aligned gigantic page 'base'. Handle any discontiguity
244 * in the mem_map at MAX_ORDER_NR_PAGES boundaries.
245 */
246static inline struct page *mem_map_offset(struct page *base, int offset)
247{
248 if (unlikely(offset >= MAX_ORDER_NR_PAGES))
249 return pfn_to_page(page_to_pfn(base) + offset);
250 return base + offset;
251}
252
253/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300254 * Iterator over all subpages within the maximally aligned gigantic
Andy Whitcroft69d177c2008-11-06 12:53:26 -0800255 * page 'base'. Handle any discontiguity in the mem_map.
256 */
257static inline struct page *mem_map_next(struct page *iter,
258 struct page *base, int offset)
259{
260 if (unlikely((offset & (MAX_ORDER_NR_PAGES - 1)) == 0)) {
261 unsigned long pfn = page_to_pfn(base) + offset;
262 if (!pfn_valid(pfn))
263 return NULL;
264 return pfn_to_page(pfn);
265 }
266 return iter + 1;
267}
268
269/*
Alexander van Heukelumb5a0e012008-02-23 15:24:06 -0800270 * FLATMEM and DISCONTIGMEM configurations use alloc_bootmem_node,
271 * so all functions starting at paging_init should be marked __init
272 * in those cases. SPARSEMEM, however, allows for memory hotplug,
273 * and alloc_bootmem_node is not used.
274 */
275#ifdef CONFIG_SPARSEMEM
276#define __paginginit __meminit
277#else
278#define __paginginit __init
279#endif
280
Mel Gorman6b74ab92008-07-23 21:26:49 -0700281/* Memory initialisation debug and verification */
282enum mminit_level {
283 MMINIT_WARNING,
284 MMINIT_VERIFY,
285 MMINIT_TRACE
286};
287
288#ifdef CONFIG_DEBUG_MEMORY_INIT
289
290extern int mminit_loglevel;
291
292#define mminit_dprintk(level, prefix, fmt, arg...) \
293do { \
294 if (level < mminit_loglevel) { \
295 printk(level <= MMINIT_WARNING ? KERN_WARNING : KERN_DEBUG); \
296 printk(KERN_CONT "mminit::" prefix " " fmt, ##arg); \
297 } \
298} while (0)
299
Mel Gorman708614e2008-07-23 21:26:51 -0700300extern void mminit_verify_pageflags_layout(void);
301extern void mminit_verify_page_links(struct page *page,
302 enum zone_type zone, unsigned long nid, unsigned long pfn);
Mel Gorman68ad8df2008-07-23 21:26:52 -0700303extern void mminit_verify_zonelist(void);
Mel Gorman708614e2008-07-23 21:26:51 -0700304
Mel Gorman6b74ab92008-07-23 21:26:49 -0700305#else
306
307static inline void mminit_dprintk(enum mminit_level level,
308 const char *prefix, const char *fmt, ...)
309{
310}
311
Mel Gorman708614e2008-07-23 21:26:51 -0700312static inline void mminit_verify_pageflags_layout(void)
313{
314}
315
316static inline void mminit_verify_page_links(struct page *page,
317 enum zone_type zone, unsigned long nid, unsigned long pfn)
318{
319}
Mel Gorman68ad8df2008-07-23 21:26:52 -0700320
321static inline void mminit_verify_zonelist(void)
322{
323}
Mel Gorman6b74ab92008-07-23 21:26:49 -0700324#endif /* CONFIG_DEBUG_MEMORY_INIT */
Mel Gorman2dbb51c2008-07-23 21:26:52 -0700325
326/* mminit_validate_memmodel_limits is independent of CONFIG_DEBUG_MEMORY_INIT */
327#if defined(CONFIG_SPARSEMEM)
328extern void mminit_validate_memmodel_limits(unsigned long *start_pfn,
329 unsigned long *end_pfn);
330#else
331static inline void mminit_validate_memmodel_limits(unsigned long *start_pfn,
332 unsigned long *end_pfn)
333{
334}
335#endif /* CONFIG_SPARSEMEM */
336
Mel Gormanfa5e0842009-06-16 15:33:22 -0700337#define ZONE_RECLAIM_NOSCAN -2
338#define ZONE_RECLAIM_FULL -1
339#define ZONE_RECLAIM_SOME 0
340#define ZONE_RECLAIM_SUCCESS 1
Wu Fengguang7c116f22009-12-16 12:19:59 +0100341
Wu Fengguang31d3d342009-12-16 12:19:59 +0100342extern int hwpoison_filter(struct page *p);
343
Wu Fengguang7c116f22009-12-16 12:19:59 +0100344extern u32 hwpoison_filter_dev_major;
345extern u32 hwpoison_filter_dev_minor;
Wu Fengguang478c5ff2009-12-16 12:19:59 +0100346extern u64 hwpoison_filter_flags_mask;
347extern u64 hwpoison_filter_flags_value;
Andi Kleen4fd466e2009-12-16 12:19:59 +0100348extern u64 hwpoison_filter_memcg;
Haicheng Li1bfe5fe2009-12-16 12:19:59 +0100349extern u32 hwpoison_filter_enable;
Al Viroeb36c582012-05-30 20:17:35 -0400350
351extern unsigned long vm_mmap_pgoff(struct file *, unsigned long,
352 unsigned long, unsigned long,
353 unsigned long, unsigned long);
Xishi Qiuca57df72012-07-31 16:43:19 -0700354
355extern void set_pageblock_order(void);
Minchan Kim02c6de82012-10-08 16:31:55 -0700356unsigned long reclaim_clean_pages_from_list(struct zone *zone,
357 struct list_head *page_list);
Bartlomiej Zolnierkiewiczd95ea5d2012-10-08 16:32:05 -0700358/* The ALLOC_WMARK bits are used as an index to zone->watermark */
359#define ALLOC_WMARK_MIN WMARK_MIN
360#define ALLOC_WMARK_LOW WMARK_LOW
361#define ALLOC_WMARK_HIGH WMARK_HIGH
362#define ALLOC_NO_WATERMARKS 0x04 /* don't check watermarks at all */
363
364/* Mask to get the watermark bits */
365#define ALLOC_WMARK_MASK (ALLOC_NO_WATERMARKS-1)
366
367#define ALLOC_HARDER 0x10 /* try to alloc harder */
368#define ALLOC_HIGH 0x20 /* __GFP_HIGH set */
369#define ALLOC_CPUSET 0x40 /* check for correct cpuset */
370#define ALLOC_CMA 0x80 /* allow allocations from CMA areas */
371
Michel Lespinassedb971412012-10-08 16:29:34 -0700372#endif /* __MM_INTERNAL_H */