blob: 473abbda942e0a1db072a3300db46211e12322fe [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef _LINUX_MM_H
2#define _LINUX_MM_H
3
Linus Torvalds1da177e2005-04-16 15:20:36 -07004#include <linux/errno.h>
5
6#ifdef __KERNEL__
7
Linus Torvalds1da177e2005-04-16 15:20:36 -07008#include <linux/gfp.h>
Paul Gortmaker187f1882011-11-23 20:12:59 -05009#include <linux/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070010#include <linux/list.h>
11#include <linux/mmzone.h>
12#include <linux/rbtree.h>
Konstantin Khlebnikov83aeead2011-12-08 14:33:54 -080013#include <linux/atomic.h>
Ingo Molnar9a11b49a2006-07-03 00:24:33 -070014#include <linux/debug_locks.h>
Heiko Carstens5b99cd02006-09-27 01:50:01 -070015#include <linux/mm_types.h>
Yinghai Lu08677212010-02-10 01:20:20 -080016#include <linux/range.h>
Chris Metcalfc6f6b592010-05-24 14:32:53 -070017#include <linux/pfn.h>
Andrea Arcangelie9da73d2011-01-13 15:46:32 -080018#include <linux/bit_spinlock.h>
Dave Chinnerb0d40c92011-07-08 14:14:42 +100019#include <linux/shrinker.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070020
21struct mempolicy;
22struct anon_vma;
Michel Lespinassebf181b92012-10-08 16:31:39 -070023struct anon_vma_chain;
Alexey Dobriyan4e950f62007-07-30 02:36:13 +040024struct file_ra_state;
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040025struct user_struct;
Alexey Dobriyan4e950f62007-07-30 02:36:13 +040026struct writeback_control;
Linus Torvalds1da177e2005-04-16 15:20:36 -070027
28#ifndef CONFIG_DISCONTIGMEM /* Don't use mapnrs, do it properly */
29extern unsigned long max_mapnr;
30#endif
31
32extern unsigned long num_physpages;
Jan Beulich44813742009-09-21 17:03:05 -070033extern unsigned long totalram_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -070034extern void * high_memory;
Linus Torvalds1da177e2005-04-16 15:20:36 -070035extern int page_cluster;
36
37#ifdef CONFIG_SYSCTL
38extern int sysctl_legacy_va_layout;
39#else
40#define sysctl_legacy_va_layout 0
41#endif
42
43#include <asm/page.h>
44#include <asm/pgtable.h>
45#include <asm/processor.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070046
Linus Torvalds1da177e2005-04-16 15:20:36 -070047#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
48
Andrea Righi27ac7922008-07-23 21:28:13 -070049/* to align the pointer to the (next) page boundary */
50#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
51
Linus Torvalds1da177e2005-04-16 15:20:36 -070052/*
53 * Linux kernel virtual memory manager primitives.
54 * The idea being to have a "virtual" mm in the same way
55 * we have a virtual fs - giving a cleaner interface to the
56 * mm details, and allowing different kinds of memory mappings
57 * (from shared memory to executable loading to arbitrary
58 * mmap() functions).
59 */
60
Christoph Lameterc43692e2006-12-06 20:32:48 -080061extern struct kmem_cache *vm_area_cachep;
62
Linus Torvalds1da177e2005-04-16 15:20:36 -070063#ifndef CONFIG_MMU
David Howells8feae132009-01-08 12:04:47 +000064extern struct rb_root nommu_region_tree;
65extern struct rw_semaphore nommu_region_sem;
Linus Torvalds1da177e2005-04-16 15:20:36 -070066
67extern unsigned int kobjsize(const void *objp);
68#endif
69
70/*
Hugh Dickins605d9282008-08-16 11:07:21 +010071 * vm_flags in vm_area_struct, see mm_types.h.
Linus Torvalds1da177e2005-04-16 15:20:36 -070072 */
Konstantin Khlebnikovcc2383e2012-10-08 16:28:37 -070073#define VM_NONE 0x00000000
74
Linus Torvalds1da177e2005-04-16 15:20:36 -070075#define VM_READ 0x00000001 /* currently active flags */
76#define VM_WRITE 0x00000002
77#define VM_EXEC 0x00000004
78#define VM_SHARED 0x00000008
79
Paolo 'Blaisorblade' Giarrusso7e2cff42005-09-21 09:55:39 -070080/* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
Linus Torvalds1da177e2005-04-16 15:20:36 -070081#define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */
82#define VM_MAYWRITE 0x00000020
83#define VM_MAYEXEC 0x00000040
84#define VM_MAYSHARE 0x00000080
85
86#define VM_GROWSDOWN 0x00000100 /* general info on the segment */
Linus Torvalds6aab3412005-11-28 14:34:23 -080087#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
Linus Torvalds1da177e2005-04-16 15:20:36 -070088#define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */
89
Michel Lespinasse18693052013-02-22 16:32:46 -080090#define VM_POPULATE 0x00001000
Linus Torvalds1da177e2005-04-16 15:20:36 -070091#define VM_LOCKED 0x00002000
92#define VM_IO 0x00004000 /* Memory mapped I/O or similar */
93
94 /* Used by sys_madvise() */
95#define VM_SEQ_READ 0x00008000 /* App will access data sequentially */
96#define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */
97
98#define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */
99#define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100#define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */
Andy Whitcroftcdfd4322008-07-23 21:27:28 -0700101#define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102#define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
103#define VM_NONLINEAR 0x00800000 /* Is non-linear (remap_file_pages) */
Konstantin Khlebnikovcc2383e2012-10-08 16:28:37 -0700104#define VM_ARCH_1 0x01000000 /* Architecture-specific flag */
Konstantin Khlebnikov0103bd12012-10-08 16:28:59 -0700105#define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700106
Jared Hulbertb379d792008-04-28 02:12:58 -0700107#define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
Konstantin Khlebnikovcc2383e2012-10-08 16:28:37 -0700108#define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */
109#define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700110#define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */
Nick Piggind00806b2007-07-19 01:46:57 -0700111
Konstantin Khlebnikovcc2383e2012-10-08 16:28:37 -0700112#if defined(CONFIG_X86)
113# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
114#elif defined(CONFIG_PPC)
115# define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */
116#elif defined(CONFIG_PARISC)
117# define VM_GROWSUP VM_ARCH_1
118#elif defined(CONFIG_IA64)
119# define VM_GROWSUP VM_ARCH_1
120#elif !defined(CONFIG_MMU)
121# define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */
122#endif
123
124#ifndef VM_GROWSUP
125# define VM_GROWSUP VM_NONE
126#endif
127
Mel Gormana8bef8ff2010-05-24 14:32:24 -0700128/* Bits set in the VMA until the stack is in its final location */
129#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ)
130
Linus Torvalds1da177e2005-04-16 15:20:36 -0700131#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */
132#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
133#endif
134
135#ifdef CONFIG_STACK_GROWSUP
136#define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
137#else
138#define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
139#endif
140
141#define VM_READHINTMASK (VM_SEQ_READ | VM_RAND_READ)
142#define VM_ClearReadHint(v) (v)->vm_flags &= ~VM_READHINTMASK
143#define VM_NormalReadHint(v) (!((v)->vm_flags & VM_READHINTMASK))
144#define VM_SequentialReadHint(v) ((v)->vm_flags & VM_SEQ_READ)
145#define VM_RandomReadHint(v) ((v)->vm_flags & VM_RAND_READ)
146
147/*
Andrea Arcangeli78f11a22011-04-27 15:26:45 -0700148 * Special vmas that are non-mergable, non-mlock()able.
149 * Note: mm/huge_memory.c VM_NO_THP depends on this definition.
Nick Pigginb291f002008-10-18 20:26:44 -0700150 */
Konstantin Khlebnikov314e51b2012-10-08 16:29:02 -0700151#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP)
Nick Pigginb291f002008-10-18 20:26:44 -0700152
153/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700154 * mapping from the currently active vm_flags protection bits (the
155 * low four bits) to a page protection mask..
156 */
157extern pgprot_t protection_map[16];
158
Nick Piggind0217ac2007-07-19 01:47:03 -0700159#define FAULT_FLAG_WRITE 0x01 /* Fault was a write access */
160#define FAULT_FLAG_NONLINEAR 0x02 /* Fault was via a nonlinear mapping */
Nick Pigginc2ec1752009-03-31 15:23:21 -0700161#define FAULT_FLAG_MKWRITE 0x04 /* Fault was mkwrite of existing pte */
Michel Lespinassed065bd82010-10-26 14:21:57 -0700162#define FAULT_FLAG_ALLOW_RETRY 0x08 /* Retry fault if blocking */
Gleb Natapov318b2752011-03-22 16:30:51 -0700163#define FAULT_FLAG_RETRY_NOWAIT 0x10 /* Don't drop mmap_sem and wait when retrying */
KOSAKI Motohiro37b23e02011-05-24 17:11:30 -0700164#define FAULT_FLAG_KILLABLE 0x20 /* The fault task is in SIGKILL killable region */
Shaohua Li45cac652012-10-08 16:32:19 -0700165#define FAULT_FLAG_TRIED 0x40 /* second try */
Nick Piggind0217ac2007-07-19 01:47:03 -0700166
venkatesh.pallipadi@intel.com6bd9cd52008-12-19 13:47:26 -0800167/*
Nick Piggind0217ac2007-07-19 01:47:03 -0700168 * vm_fault is filled by the the pagefault handler and passed to the vma's
Nick Piggin83c54072007-07-19 01:47:05 -0700169 * ->fault function. The vma's ->fault is responsible for returning a bitmask
170 * of VM_FAULT_xxx flags that give details about how the fault was handled.
Nick Piggin54cb8822007-07-19 01:46:59 -0700171 *
Nick Piggind0217ac2007-07-19 01:47:03 -0700172 * pgoff should be used in favour of virtual_address, if possible. If pgoff
Konstantin Khlebnikov0b173bc2012-10-08 16:28:46 -0700173 * is used, one may implement ->remap_pages to get nonlinear mapping support.
Nick Piggin54cb8822007-07-19 01:46:59 -0700174 */
Nick Piggind0217ac2007-07-19 01:47:03 -0700175struct vm_fault {
176 unsigned int flags; /* FAULT_FLAG_xxx flags */
177 pgoff_t pgoff; /* Logical page offset based on vma */
178 void __user *virtual_address; /* Faulting virtual address */
179
180 struct page *page; /* ->fault handlers should return a
Nick Piggin83c54072007-07-19 01:47:05 -0700181 * page here, unless VM_FAULT_NOPAGE
Nick Piggind0217ac2007-07-19 01:47:03 -0700182 * is set (which is also implied by
Nick Piggin83c54072007-07-19 01:47:05 -0700183 * VM_FAULT_ERROR).
Nick Piggind0217ac2007-07-19 01:47:03 -0700184 */
Nick Piggin54cb8822007-07-19 01:46:59 -0700185};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186
187/*
188 * These are the virtual MM functions - opening of an area, closing and
189 * unmapping it (needed to keep files on disk up-to-date etc), pointer
190 * to the functions called when a no-page or a wp-page exception occurs.
191 */
192struct vm_operations_struct {
193 void (*open)(struct vm_area_struct * area);
194 void (*close)(struct vm_area_struct * area);
Nick Piggind0217ac2007-07-19 01:47:03 -0700195 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
David Howells9637a5e2006-06-23 02:03:43 -0700196
197 /* notification that a previously read-only page is about to become
198 * writable, if an error is returned it will cause a SIGBUS */
Nick Pigginc2ec1752009-03-31 15:23:21 -0700199 int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
Rik van Riel28b2ee22008-07-23 21:27:05 -0700200
201 /* called by access_process_vm when get_user_pages() fails, typically
202 * for use by special VMAs that can switch between memory and hardware
203 */
204 int (*access)(struct vm_area_struct *vma, unsigned long addr,
205 void *buf, int len, int write);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700206#ifdef CONFIG_NUMA
Lee Schermerhorna6020ed2008-04-28 02:13:14 -0700207 /*
208 * set_policy() op must add a reference to any non-NULL @new mempolicy
209 * to hold the policy upon return. Caller should pass NULL @new to
210 * remove a policy and fall back to surrounding context--i.e. do not
211 * install a MPOL_DEFAULT policy, nor the task or system default
212 * mempolicy.
213 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700214 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
Lee Schermerhorna6020ed2008-04-28 02:13:14 -0700215
216 /*
217 * get_policy() op must add reference [mpol_get()] to any policy at
218 * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure
219 * in mm/mempolicy.c will do this automatically.
220 * get_policy() must NOT add a ref if the policy at (vma,addr) is not
221 * marked as MPOL_SHARED. vma policies are protected by the mmap_sem.
222 * If no [shared/vma] mempolicy exists at the addr, get_policy() op
223 * must return NULL--i.e., do not "fallback" to task or system default
224 * policy.
225 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700226 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
227 unsigned long addr);
Christoph Lameter7b2259b2006-06-25 05:46:48 -0700228 int (*migrate)(struct vm_area_struct *vma, const nodemask_t *from,
229 const nodemask_t *to, unsigned long flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230#endif
Konstantin Khlebnikov0b173bc2012-10-08 16:28:46 -0700231 /* called by sys_remap_file_pages() to populate non-linear mapping */
232 int (*remap_pages)(struct vm_area_struct *vma, unsigned long addr,
233 unsigned long size, pgoff_t pgoff);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700234};
235
236struct mmu_gather;
237struct inode;
238
Andrew Morton349aef02006-01-08 01:04:36 -0800239#define page_private(page) ((page)->private)
240#define set_page_private(page, v) ((page)->private = (v))
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700241
Minchan Kimb12c4ad2012-10-08 16:32:08 -0700242/* It's valid only if the page is free path or free_list */
243static inline void set_freepage_migratetype(struct page *page, int migratetype)
244{
Minchan Kim95e34412012-10-08 16:32:11 -0700245 page->index = migratetype;
Minchan Kimb12c4ad2012-10-08 16:32:08 -0700246}
247
248/* It's valid only if the page is free path or free_list */
249static inline int get_freepage_migratetype(struct page *page)
250{
Minchan Kim95e34412012-10-08 16:32:11 -0700251 return page->index;
Minchan Kimb12c4ad2012-10-08 16:32:08 -0700252}
253
Linus Torvalds1da177e2005-04-16 15:20:36 -0700254/*
255 * FIXME: take this include out, include page-flags.h in
256 * files which need it (119 of them)
257 */
258#include <linux/page-flags.h>
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800259#include <linux/huge_mm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700260
261/*
262 * Methods to modify the page usage count.
263 *
264 * What counts for a page usage:
265 * - cache mapping (page->mapping)
266 * - private data (page->private)
267 * - page mapped in a task's page tables, each mapping
268 * is counted separately
269 *
270 * Also, many kernel routines increase the page count before a critical
271 * routine so they can be sure the page doesn't go away from under them.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700272 */
273
274/*
Nick Pigginda6052f2006-09-25 23:31:35 -0700275 * Drop a ref, return true if the refcount fell to zero (the page has no users)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276 */
Nick Piggin7c8ee9a2006-03-22 00:08:03 -0800277static inline int put_page_testzero(struct page *page)
278{
Nick Piggin725d7042006-09-25 23:30:55 -0700279 VM_BUG_ON(atomic_read(&page->_count) == 0);
Nick Piggin8dc04ef2006-03-22 00:08:03 -0800280 return atomic_dec_and_test(&page->_count);
Nick Piggin7c8ee9a2006-03-22 00:08:03 -0800281}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700282
283/*
Nick Piggin7c8ee9a2006-03-22 00:08:03 -0800284 * Try to grab a ref unless the page has a refcount of zero, return false if
285 * that is the case.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286 */
Nick Piggin7c8ee9a2006-03-22 00:08:03 -0800287static inline int get_page_unless_zero(struct page *page)
288{
Nick Piggin8dc04ef2006-03-22 00:08:03 -0800289 return atomic_inc_not_zero(&page->_count);
Nick Piggin7c8ee9a2006-03-22 00:08:03 -0800290}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291
Wu Fengguang53df8fd2010-01-27 11:06:39 +0800292extern int page_is_ram(unsigned long pfn);
293
Christoph Lameter48667e72008-02-04 22:28:31 -0800294/* Support for virtually mapped pages */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800295struct page *vmalloc_to_page(const void *addr);
296unsigned long vmalloc_to_pfn(const void *addr);
Christoph Lameter48667e72008-02-04 22:28:31 -0800297
Paul Mundt0738c4b2008-03-12 16:51:31 +0900298/*
299 * Determine if an address is within the vmalloc range
300 *
301 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
302 * is no special casing required.
303 */
Christoph Lameter9e2779f2008-02-04 22:28:34 -0800304static inline int is_vmalloc_addr(const void *x)
305{
Paul Mundt0738c4b2008-03-12 16:51:31 +0900306#ifdef CONFIG_MMU
Christoph Lameter9e2779f2008-02-04 22:28:34 -0800307 unsigned long addr = (unsigned long)x;
308
309 return addr >= VMALLOC_START && addr < VMALLOC_END;
Paul Mundt0738c4b2008-03-12 16:51:31 +0900310#else
311 return 0;
David Howells8ca3ed82008-02-23 15:23:37 -0800312#endif
Paul Mundt0738c4b2008-03-12 16:51:31 +0900313}
KAMEZAWA Hiroyuki81ac3ad2009-09-22 16:45:49 -0700314#ifdef CONFIG_MMU
315extern int is_vmalloc_or_module_addr(const void *x);
316#else
David Howells934831d2009-09-24 12:33:32 +0100317static inline int is_vmalloc_or_module_addr(const void *x)
KAMEZAWA Hiroyuki81ac3ad2009-09-22 16:45:49 -0700318{
319 return 0;
320}
321#endif
Christoph Lameter9e2779f2008-02-04 22:28:34 -0800322
Andrea Arcangelie9da73d2011-01-13 15:46:32 -0800323static inline void compound_lock(struct page *page)
324{
325#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Pravin B Shelar5bf5f032012-05-29 15:06:49 -0700326 VM_BUG_ON(PageSlab(page));
Andrea Arcangelie9da73d2011-01-13 15:46:32 -0800327 bit_spin_lock(PG_compound_lock, &page->flags);
328#endif
329}
330
331static inline void compound_unlock(struct page *page)
332{
333#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Pravin B Shelar5bf5f032012-05-29 15:06:49 -0700334 VM_BUG_ON(PageSlab(page));
Andrea Arcangelie9da73d2011-01-13 15:46:32 -0800335 bit_spin_unlock(PG_compound_lock, &page->flags);
336#endif
337}
338
339static inline unsigned long compound_lock_irqsave(struct page *page)
340{
341 unsigned long uninitialized_var(flags);
342#ifdef CONFIG_TRANSPARENT_HUGEPAGE
343 local_irq_save(flags);
344 compound_lock(page);
345#endif
346 return flags;
347}
348
349static inline void compound_unlock_irqrestore(struct page *page,
350 unsigned long flags)
351{
352#ifdef CONFIG_TRANSPARENT_HUGEPAGE
353 compound_unlock(page);
354 local_irq_restore(flags);
355#endif
356}
357
Christoph Lameterd85f3382007-05-06 14:49:39 -0700358static inline struct page *compound_head(struct page *page)
359{
Christoph Lameter6d777952007-05-06 14:49:40 -0700360 if (unlikely(PageTail(page)))
Christoph Lameterd85f3382007-05-06 14:49:39 -0700361 return page->first_page;
362 return page;
363}
364
Andrea Arcangeli70b50f92011-11-02 13:36:59 -0700365/*
366 * The atomic page->_mapcount, starts from -1: so that transitions
367 * both from it and to it can be tracked, using atomic_inc_and_test
368 * and atomic_add_negative(-1).
369 */
370static inline void reset_page_mapcount(struct page *page)
371{
372 atomic_set(&(page)->_mapcount, -1);
373}
374
375static inline int page_mapcount(struct page *page)
376{
377 return atomic_read(&(page)->_mapcount) + 1;
378}
379
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700380static inline int page_count(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381{
Christoph Lameterd85f3382007-05-06 14:49:39 -0700382 return atomic_read(&compound_head(page)->_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700383}
384
Andrea Arcangelib35a35b2011-11-02 13:37:36 -0700385static inline void get_huge_page_tail(struct page *page)
386{
387 /*
388 * __split_huge_page_refcount() cannot run
389 * from under us.
390 */
391 VM_BUG_ON(page_mapcount(page) < 0);
392 VM_BUG_ON(atomic_read(&page->_count) != 0);
393 atomic_inc(&page->_mapcount);
394}
395
Andrea Arcangeli70b50f92011-11-02 13:36:59 -0700396extern bool __get_page_tail(struct page *page);
397
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398static inline void get_page(struct page *page)
399{
Andrea Arcangeli70b50f92011-11-02 13:36:59 -0700400 if (unlikely(PageTail(page)))
401 if (likely(__get_page_tail(page)))
402 return;
Andrea Arcangeli91807062011-01-13 15:46:32 -0800403 /*
404 * Getting a normal page or the head of a compound page
Andrea Arcangeli70b50f92011-11-02 13:36:59 -0700405 * requires to already have an elevated page->_count.
Andrea Arcangeli91807062011-01-13 15:46:32 -0800406 */
Andrea Arcangeli70b50f92011-11-02 13:36:59 -0700407 VM_BUG_ON(atomic_read(&page->_count) <= 0);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408 atomic_inc(&page->_count);
409}
410
Christoph Lameterb49af682007-05-06 14:49:41 -0700411static inline struct page *virt_to_head_page(const void *x)
412{
413 struct page *page = virt_to_page(x);
414 return compound_head(page);
415}
416
Nick Piggin7835e982006-03-22 00:08:40 -0800417/*
418 * Setup the page count before being freed into the page allocator for
419 * the first time (boot or memory hotplug)
420 */
421static inline void init_page_count(struct page *page)
422{
423 atomic_set(&page->_count, 1);
424}
425
Andrea Arcangeli5f24ce52011-01-13 15:47:00 -0800426/*
427 * PageBuddy() indicate that the page is free and in the buddy system
428 * (see mm/page_alloc.c).
Andrea Arcangelief2b4b92011-03-18 00:16:35 +0100429 *
430 * PAGE_BUDDY_MAPCOUNT_VALUE must be <= -2 but better not too close to
431 * -2 so that an underflow of the page_mapcount() won't be mistaken
432 * for a genuine PAGE_BUDDY_MAPCOUNT_VALUE. -128 can be created very
433 * efficiently by most CPU architectures.
Andrea Arcangeli5f24ce52011-01-13 15:47:00 -0800434 */
Andrea Arcangelief2b4b92011-03-18 00:16:35 +0100435#define PAGE_BUDDY_MAPCOUNT_VALUE (-128)
436
Andrea Arcangeli5f24ce52011-01-13 15:47:00 -0800437static inline int PageBuddy(struct page *page)
438{
Andrea Arcangelief2b4b92011-03-18 00:16:35 +0100439 return atomic_read(&page->_mapcount) == PAGE_BUDDY_MAPCOUNT_VALUE;
Andrea Arcangeli5f24ce52011-01-13 15:47:00 -0800440}
441
442static inline void __SetPageBuddy(struct page *page)
443{
444 VM_BUG_ON(atomic_read(&page->_mapcount) != -1);
Andrea Arcangelief2b4b92011-03-18 00:16:35 +0100445 atomic_set(&page->_mapcount, PAGE_BUDDY_MAPCOUNT_VALUE);
Andrea Arcangeli5f24ce52011-01-13 15:47:00 -0800446}
447
448static inline void __ClearPageBuddy(struct page *page)
449{
450 VM_BUG_ON(!PageBuddy(page));
451 atomic_set(&page->_mapcount, -1);
452}
453
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454void put_page(struct page *page);
Alexander Zarochentsev1d7ea732006-08-13 23:24:27 -0700455void put_pages_list(struct list_head *pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700456
Nick Piggin8dfcc9b2006-03-22 00:08:05 -0800457void split_page(struct page *page, unsigned int order);
Mel Gorman748446b2010-05-24 14:32:27 -0700458int split_free_page(struct page *page);
Nick Piggin8dfcc9b2006-03-22 00:08:05 -0800459
Linus Torvalds1da177e2005-04-16 15:20:36 -0700460/*
Andy Whitcroft33f2ef82006-12-06 20:33:32 -0800461 * Compound pages have a destructor function. Provide a
462 * prototype for that function and accessor functions.
463 * These are _only_ valid on the head of a PG_compound page.
464 */
465typedef void compound_page_dtor(struct page *);
466
467static inline void set_compound_page_dtor(struct page *page,
468 compound_page_dtor *dtor)
469{
470 page[1].lru.next = (void *)dtor;
471}
472
473static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
474{
475 return (compound_page_dtor *)page[1].lru.next;
476}
477
Christoph Lameterd85f3382007-05-06 14:49:39 -0700478static inline int compound_order(struct page *page)
479{
Christoph Lameter6d777952007-05-06 14:49:40 -0700480 if (!PageHead(page))
Christoph Lameterd85f3382007-05-06 14:49:39 -0700481 return 0;
482 return (unsigned long)page[1].lru.prev;
483}
484
Andrea Arcangeli37c2ac72011-01-13 15:47:16 -0800485static inline int compound_trans_order(struct page *page)
486{
487 int order;
488 unsigned long flags;
489
490 if (!PageHead(page))
491 return 0;
492
493 flags = compound_lock_irqsave(page);
494 order = compound_order(page);
495 compound_unlock_irqrestore(page, flags);
496 return order;
497}
498
Christoph Lameterd85f3382007-05-06 14:49:39 -0700499static inline void set_compound_order(struct page *page, unsigned long order)
500{
501 page[1].lru.prev = (void *)order;
502}
503
Michal Simek3dece372011-01-21 08:49:56 +0100504#ifdef CONFIG_MMU
Andy Whitcroft33f2ef82006-12-06 20:33:32 -0800505/*
Andrea Arcangeli14fd4032011-01-13 15:46:37 -0800506 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
507 * servicing faults for write access. In the normal case, do always want
508 * pte_mkwrite. But get_user_pages can cause write faults for mappings
509 * that do not have writing enabled, when used by access_process_vm.
510 */
511static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
512{
513 if (likely(vma->vm_flags & VM_WRITE))
514 pte = pte_mkwrite(pte);
515 return pte;
516}
Michal Simek3dece372011-01-21 08:49:56 +0100517#endif
Andrea Arcangeli14fd4032011-01-13 15:46:37 -0800518
519/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700520 * Multiple processes may "see" the same page. E.g. for untouched
521 * mappings of /dev/null, all processes see the same page full of
522 * zeroes, and text pages of executables and shared libraries have
523 * only one copy in memory, at most, normally.
524 *
525 * For the non-reserved pages, page_count(page) denotes a reference count.
Paolo 'Blaisorblade' Giarrusso7e871b62005-09-21 09:55:38 -0700526 * page_count() == 0 means the page is free. page->lru is then used for
527 * freelist management in the buddy allocator.
Nick Pigginda6052f2006-09-25 23:31:35 -0700528 * page_count() > 0 means the page has been allocated.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700529 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700530 * Pages are allocated by the slab allocator in order to provide memory
531 * to kmalloc and kmem_cache_alloc. In this case, the management of the
532 * page, and the fields in 'struct page' are the responsibility of mm/slab.c
533 * unless a particular usage is carefully commented. (the responsibility of
534 * freeing the kmalloc memory is the caller's, of course).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700536 * A page may be used by anyone else who does a __get_free_page().
537 * In this case, page_count still tracks the references, and should only
538 * be used through the normal accessor functions. The top bits of page->flags
539 * and page->virtual store page management information, but all other fields
540 * are unused and could be used privately, carefully. The management of this
541 * page is the responsibility of the one who allocated it, and those who have
542 * subsequently been given references to it.
543 *
544 * The other pages (we may call them "pagecache pages") are completely
Linus Torvalds1da177e2005-04-16 15:20:36 -0700545 * managed by the Linux memory manager: I/O, buffers, swapping etc.
546 * The following discussion applies only to them.
547 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700548 * A pagecache page contains an opaque `private' member, which belongs to the
549 * page's address_space. Usually, this is the address of a circular list of
550 * the page's disk buffers. PG_private must be set to tell the VM to call
551 * into the filesystem to release these pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700552 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700553 * A page may belong to an inode's memory mapping. In this case, page->mapping
554 * is the pointer to the inode, and page->index is the file offset of the page,
555 * in units of PAGE_CACHE_SIZE.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700557 * If pagecache pages are not associated with an inode, they are said to be
558 * anonymous pages. These may become associated with the swapcache, and in that
559 * case PG_swapcache is set, and page->private is an offset into the swapcache.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700560 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700561 * In either case (swapcache or inode backed), the pagecache itself holds one
562 * reference to the page. Setting PG_private should also increment the
563 * refcount. The each user mapping also has a reference to the page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700565 * The pagecache pages are stored in a per-mapping radix tree, which is
566 * rooted at mapping->page_tree, and indexed by offset.
567 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
568 * lists, we instead now tag pages as dirty/writeback in the radix tree.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700570 * All pagecache pages may be subject to I/O:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571 * - inode pages may need to be read from disk,
572 * - inode pages which have been modified and are MAP_SHARED may need
Nick Pigginda6052f2006-09-25 23:31:35 -0700573 * to be written back to the inode on disk,
574 * - anonymous pages (including MAP_PRIVATE file mappings) which have been
575 * modified may need to be swapped out to swap space and (later) to be read
576 * back into memory.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700577 */
578
579/*
580 * The zone field is never updated after free_area_init_core()
581 * sets it, so none of the operations on it need to be atomic.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700582 */
Dave Hansen348f8b62005-06-23 00:07:40 -0700583
Peter Zijlstra75980e92013-02-22 16:34:32 -0800584/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_NID] | ... | FLAGS | */
Andi Kleen07808b72005-11-05 17:25:53 +0100585#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700586#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
587#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
Peter Zijlstra75980e92013-02-22 16:34:32 -0800588#define LAST_NID_PGOFF (ZONES_PGOFF - LAST_NID_WIDTH)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700589
590/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300591 * Define the bit shifts to access each section. For non-existent
Dave Hansen348f8b62005-06-23 00:07:40 -0700592 * sections we define the shift as 0; that plus a 0 mask ensures
593 * the compiler will optimise away reference to them.
594 */
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700595#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
596#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
597#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
Peter Zijlstra75980e92013-02-22 16:34:32 -0800598#define LAST_NID_PGSHIFT (LAST_NID_PGOFF * (LAST_NID_WIDTH != 0))
Dave Hansen348f8b62005-06-23 00:07:40 -0700599
Will Deaconbce54bb2010-10-26 14:21:37 -0700600/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
601#ifdef NODE_NOT_IN_PAGE_FLAGS
Christoph Lameter89689ae2006-12-06 20:31:45 -0800602#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
Andy Whitcroftbd8029b2007-02-10 01:43:14 -0800603#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
604 SECTIONS_PGOFF : ZONES_PGOFF)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700605#else
Christoph Lameter89689ae2006-12-06 20:31:45 -0800606#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
Andy Whitcroftbd8029b2007-02-10 01:43:14 -0800607#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
608 NODES_PGOFF : ZONES_PGOFF)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700609#endif
Christoph Lameter89689ae2006-12-06 20:31:45 -0800610
Andy Whitcroftbd8029b2007-02-10 01:43:14 -0800611#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
Dave Hansen348f8b62005-06-23 00:07:40 -0700612
Christoph Lameter9223b4192008-04-28 02:12:48 -0700613#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
614#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
Dave Hansen348f8b62005-06-23 00:07:40 -0700615#endif
616
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700617#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
618#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
619#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
Peter Zijlstra75980e92013-02-22 16:34:32 -0800620#define LAST_NID_MASK ((1UL << LAST_NID_WIDTH) - 1)
Christoph Lameter89689ae2006-12-06 20:31:45 -0800621#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
Dave Hansen348f8b62005-06-23 00:07:40 -0700622
Ian Campbell33dd4e02011-07-25 17:11:51 -0700623static inline enum zone_type page_zonenum(const struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700624{
Dave Hansen348f8b62005-06-23 00:07:40 -0700625 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700626}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700627
Christoph Lameter89689ae2006-12-06 20:31:45 -0800628/*
629 * The identification function is only used by the buddy allocator for
630 * determining if two pages could be buddies. We are not really
631 * identifying a zone since we could be using a the section number
632 * id if we have not node id available in page flags.
633 * We guarantee only that it will return the same value for two
634 * combinable pages in a zone.
635 */
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -0700636static inline int page_zone_id(struct page *page)
637{
Christoph Lameter89689ae2006-12-06 20:31:45 -0800638 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639}
640
Andy Whitcroft25ba77c2006-12-06 20:33:03 -0800641static inline int zone_to_nid(struct zone *zone)
Christoph Lameter89fa3022006-09-25 23:31:55 -0700642{
Christoph Lameterd5f541e2006-09-27 01:50:08 -0700643#ifdef CONFIG_NUMA
644 return zone->node;
645#else
646 return 0;
647#endif
Christoph Lameter89fa3022006-09-25 23:31:55 -0700648}
649
Christoph Lameter89689ae2006-12-06 20:31:45 -0800650#ifdef NODE_NOT_IN_PAGE_FLAGS
Ian Campbell33dd4e02011-07-25 17:11:51 -0700651extern int page_to_nid(const struct page *page);
Christoph Lameter89689ae2006-12-06 20:31:45 -0800652#else
Ian Campbell33dd4e02011-07-25 17:11:51 -0700653static inline int page_to_nid(const struct page *page)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700654{
Christoph Lameter89689ae2006-12-06 20:31:45 -0800655 return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700656}
Christoph Lameter89689ae2006-12-06 20:31:45 -0800657#endif
658
Mel Gorman57e0a032012-11-12 09:06:20 +0000659#ifdef CONFIG_NUMA_BALANCING
Peter Zijlstra75980e92013-02-22 16:34:32 -0800660#ifdef LAST_NID_NOT_IN_PAGE_FLAGS
Mel Gorman57e0a032012-11-12 09:06:20 +0000661static inline int page_xchg_last_nid(struct page *page, int nid)
662{
663 return xchg(&page->_last_nid, nid);
664}
665
666static inline int page_last_nid(struct page *page)
667{
668 return page->_last_nid;
669}
670static inline void reset_page_last_nid(struct page *page)
671{
672 page->_last_nid = -1;
673}
674#else
Peter Zijlstra75980e92013-02-22 16:34:32 -0800675static inline int page_last_nid(struct page *page)
676{
677 return (page->flags >> LAST_NID_PGSHIFT) & LAST_NID_MASK;
678}
679
680static inline int page_xchg_last_nid(struct page *page, int nid)
681{
682 unsigned long old_flags, flags;
683 int last_nid;
684
685 do {
686 old_flags = flags = page->flags;
687 last_nid = page_last_nid(page);
688
689 flags &= ~(LAST_NID_MASK << LAST_NID_PGSHIFT);
690 flags |= (nid & LAST_NID_MASK) << LAST_NID_PGSHIFT;
691 } while (unlikely(cmpxchg(&page->flags, old_flags, flags) != old_flags));
692
693 return last_nid;
694}
695
696static inline void reset_page_last_nid(struct page *page)
697{
698 page_xchg_last_nid(page, (1 << LAST_NID_SHIFT) - 1);
699}
700#endif /* LAST_NID_NOT_IN_PAGE_FLAGS */
701#else
Mel Gorman57e0a032012-11-12 09:06:20 +0000702static inline int page_xchg_last_nid(struct page *page, int nid)
703{
704 return page_to_nid(page);
705}
706
707static inline int page_last_nid(struct page *page)
708{
709 return page_to_nid(page);
710}
711
712static inline void reset_page_last_nid(struct page *page)
713{
714}
715#endif
716
Ian Campbell33dd4e02011-07-25 17:11:51 -0700717static inline struct zone *page_zone(const struct page *page)
Christoph Lameter89689ae2006-12-06 20:31:45 -0800718{
719 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
720}
721
Christoph Lameter308c05e2008-04-28 02:12:43 -0700722#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
Daniel Kiperbf4e8902011-05-24 17:12:32 -0700723static inline void set_page_section(struct page *page, unsigned long section)
724{
725 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
726 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
727}
728
Ian Campbellaa462ab2011-08-17 17:40:33 +0100729static inline unsigned long page_to_section(const struct page *page)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700730{
731 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
732}
Christoph Lameter308c05e2008-04-28 02:12:43 -0700733#endif
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700734
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700735static inline void set_page_zone(struct page *page, enum zone_type zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736{
Dave Hansen348f8b62005-06-23 00:07:40 -0700737 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
738 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
739}
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700740
Dave Hansen348f8b62005-06-23 00:07:40 -0700741static inline void set_page_node(struct page *page, unsigned long node)
742{
743 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
744 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
745}
Christoph Lameter89689ae2006-12-06 20:31:45 -0800746
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700747static inline void set_page_links(struct page *page, enum zone_type zone,
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700748 unsigned long node, unsigned long pfn)
Dave Hansen348f8b62005-06-23 00:07:40 -0700749{
750 set_page_zone(page, zone);
751 set_page_node(page, node);
Daniel Kiperbf4e8902011-05-24 17:12:32 -0700752#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700753 set_page_section(page, pfn_to_section_nr(pfn));
Daniel Kiperbf4e8902011-05-24 17:12:32 -0700754#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700755}
756
Christoph Lameterf6ac2352006-06-30 01:55:32 -0700757/*
758 * Some inline functions in vmstat.h depend on page_zone()
759 */
760#include <linux/vmstat.h>
761
Ian Campbell33dd4e02011-07-25 17:11:51 -0700762static __always_inline void *lowmem_page_address(const struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700763{
Ian Campbellaa462ab2011-08-17 17:40:33 +0100764 return __va(PFN_PHYS(page_to_pfn(page)));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700765}
766
767#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
768#define HASHED_PAGE_VIRTUAL
769#endif
770
771#if defined(WANT_PAGE_VIRTUAL)
772#define page_address(page) ((page)->virtual)
773#define set_page_address(page, address) \
774 do { \
775 (page)->virtual = (address); \
776 } while(0)
777#define page_address_init() do { } while(0)
778#endif
779
780#if defined(HASHED_PAGE_VIRTUAL)
Ian Campbellf9918792011-08-17 13:45:09 +0100781void *page_address(const struct page *page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700782void set_page_address(struct page *page, void *virtual);
783void page_address_init(void);
784#endif
785
786#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
787#define page_address(page) lowmem_page_address(page)
788#define set_page_address(page, address) do { } while(0)
789#define page_address_init() do { } while(0)
790#endif
791
792/*
793 * On an anonymous page mapped into a user virtual memory area,
794 * page->mapping points to its anon_vma, not to a struct address_space;
Hugh Dickins3ca7b3c2009-12-14 17:58:57 -0800795 * with the PAGE_MAPPING_ANON bit set to distinguish it. See rmap.h.
796 *
797 * On an anonymous page in a VM_MERGEABLE area, if CONFIG_KSM is enabled,
798 * the PAGE_MAPPING_KSM bit may be set along with the PAGE_MAPPING_ANON bit;
799 * and then page->mapping points, not to an anon_vma, but to a private
800 * structure which KSM associates with that merged page. See ksm.h.
801 *
802 * PAGE_MAPPING_KSM without PAGE_MAPPING_ANON is currently never used.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700803 *
804 * Please note that, confusingly, "page_mapping" refers to the inode
805 * address_space which maps the page from disk; whereas "page_mapped"
806 * refers to user virtual address space into which the page is mapped.
807 */
808#define PAGE_MAPPING_ANON 1
Hugh Dickins3ca7b3c2009-12-14 17:58:57 -0800809#define PAGE_MAPPING_KSM 2
810#define PAGE_MAPPING_FLAGS (PAGE_MAPPING_ANON | PAGE_MAPPING_KSM)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700811
812extern struct address_space swapper_space;
813static inline struct address_space *page_mapping(struct page *page)
814{
815 struct address_space *mapping = page->mapping;
816
Christoph Lameterb5fab14e2007-07-17 04:03:33 -0700817 VM_BUG_ON(PageSlab(page));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700818 if (unlikely(PageSwapCache(page)))
819 mapping = &swapper_space;
Steven Rostedte20e8772011-01-13 15:46:17 -0800820 else if ((unsigned long)mapping & PAGE_MAPPING_ANON)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700821 mapping = NULL;
822 return mapping;
823}
824
Hugh Dickins3ca7b3c2009-12-14 17:58:57 -0800825/* Neutral page->mapping pointer to address_space or anon_vma or other */
826static inline void *page_rmapping(struct page *page)
827{
828 return (void *)((unsigned long)page->mapping & ~PAGE_MAPPING_FLAGS);
829}
830
Mel Gormanf981c592012-07-31 16:44:47 -0700831extern struct address_space *__page_file_mapping(struct page *);
832
833static inline
834struct address_space *page_file_mapping(struct page *page)
835{
836 if (unlikely(PageSwapCache(page)))
837 return __page_file_mapping(page);
838
839 return page->mapping;
840}
841
Linus Torvalds1da177e2005-04-16 15:20:36 -0700842static inline int PageAnon(struct page *page)
843{
844 return ((unsigned long)page->mapping & PAGE_MAPPING_ANON) != 0;
845}
846
847/*
848 * Return the pagecache index of the passed page. Regular pagecache pages
849 * use ->index whereas swapcache pages use ->private
850 */
851static inline pgoff_t page_index(struct page *page)
852{
853 if (unlikely(PageSwapCache(page)))
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700854 return page_private(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700855 return page->index;
856}
857
Mel Gormanf981c592012-07-31 16:44:47 -0700858extern pgoff_t __page_file_index(struct page *page);
859
860/*
861 * Return the file index of the page. Regular pagecache pages use ->index
862 * whereas swapcache pages use swp_offset(->private)
863 */
864static inline pgoff_t page_file_index(struct page *page)
865{
866 if (unlikely(PageSwapCache(page)))
867 return __page_file_index(page);
868
869 return page->index;
870}
871
Linus Torvalds1da177e2005-04-16 15:20:36 -0700872/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700873 * Return true if this page is mapped into pagetables.
874 */
875static inline int page_mapped(struct page *page)
876{
877 return atomic_read(&(page)->_mapcount) >= 0;
878}
879
880/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881 * Different kinds of faults, as returned by handle_mm_fault().
882 * Used to decide whether a process gets delivered SIGBUS or
883 * just gets major/minor fault counters bumped up.
884 */
Nick Piggind0217ac2007-07-19 01:47:03 -0700885
Nick Piggin83c54072007-07-19 01:47:05 -0700886#define VM_FAULT_MINOR 0 /* For backwards compat. Remove me quickly. */
Nick Piggind0217ac2007-07-19 01:47:03 -0700887
Nick Piggin83c54072007-07-19 01:47:05 -0700888#define VM_FAULT_OOM 0x0001
889#define VM_FAULT_SIGBUS 0x0002
890#define VM_FAULT_MAJOR 0x0004
891#define VM_FAULT_WRITE 0x0008 /* Special case for get_user_pages */
Andi Kleenaa50d3a2010-10-06 21:45:00 +0200892#define VM_FAULT_HWPOISON 0x0010 /* Hit poisoned small page */
893#define VM_FAULT_HWPOISON_LARGE 0x0020 /* Hit poisoned large page. Index encoded in upper bits */
Nick Pigginf33ea7f2005-08-03 20:24:01 +1000894
Nick Piggin83c54072007-07-19 01:47:05 -0700895#define VM_FAULT_NOPAGE 0x0100 /* ->fault installed the pte, not return page */
896#define VM_FAULT_LOCKED 0x0200 /* ->fault locked the returned page */
Michel Lespinassed065bd82010-10-26 14:21:57 -0700897#define VM_FAULT_RETRY 0x0400 /* ->fault blocked, must retry */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700898
Andi Kleenaa50d3a2010-10-06 21:45:00 +0200899#define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */
900
901#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_HWPOISON | \
902 VM_FAULT_HWPOISON_LARGE)
903
904/* Encode hstate index for a hwpoisoned large page */
905#define VM_FAULT_SET_HINDEX(x) ((x) << 12)
906#define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf)
Nick Piggind0217ac2007-07-19 01:47:03 -0700907
Nick Piggin1c0fe6e2009-01-06 14:38:59 -0800908/*
909 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
910 */
911extern void pagefault_out_of_memory(void);
912
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
914
David Rientjesddd588b2011-03-22 16:30:46 -0700915/*
David Rientjes7bf02ea2011-05-24 17:11:16 -0700916 * Flags passed to show_mem() and show_free_areas() to suppress output in
David Rientjesddd588b2011-03-22 16:30:46 -0700917 * various contexts.
918 */
919#define SHOW_MEM_FILTER_NODES (0x0001u) /* filter disallowed nodes */
920
David Rientjes7bf02ea2011-05-24 17:11:16 -0700921extern void show_free_areas(unsigned int flags);
922extern bool skip_free_areas_node(unsigned int flags, int nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700923
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924int shmem_zero_setup(struct vm_area_struct *);
925
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +0400926extern int can_do_mlock(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700927extern int user_shm_lock(size_t, struct user_struct *);
928extern void user_shm_unlock(size_t, struct user_struct *);
929
930/*
931 * Parameter block passed down to zap_pte_range in exceptional cases.
932 */
933struct zap_details {
934 struct vm_area_struct *nonlinear_vma; /* Check page->index if set */
935 struct address_space *check_mapping; /* Check page->mapping if set */
936 pgoff_t first_index; /* Lowest page->index to unmap */
937 pgoff_t last_index; /* Highest page->index to unmap */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700938};
939
Nick Piggin7e675132008-04-28 02:13:00 -0700940struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
941 pte_t pte);
942
Jack Steinerc627f9c2008-07-29 22:33:53 -0700943int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
944 unsigned long size);
Al Viro14f5ff52012-03-05 13:38:09 -0500945void zap_page_range(struct vm_area_struct *vma, unsigned long address,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946 unsigned long size, struct zap_details *);
Linus Torvalds4f74d2c2012-05-06 13:54:06 -0700947void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
948 unsigned long start, unsigned long end);
Matt Mackalle6473092008-02-04 22:29:01 -0800949
950/**
951 * mm_walk - callbacks for walk_page_range
952 * @pgd_entry: if set, called for each non-empty PGD (top-level) entry
953 * @pud_entry: if set, called for each non-empty PUD (2nd-level) entry
954 * @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry
Dave Hansen03319322011-03-22 16:32:56 -0700955 * this handler is required to be able to handle
956 * pmd_trans_huge() pmds. They may simply choose to
957 * split_huge_page() instead of handling it explicitly.
Matt Mackalle6473092008-02-04 22:29:01 -0800958 * @pte_entry: if set, called for each non-empty PTE (4th-level) entry
959 * @pte_hole: if set, called for each hole at all levels
Naoya Horiguchi5dc37642009-12-14 18:00:01 -0800960 * @hugetlb_entry: if set, called for each hugetlb entry
KOSAKI Motohiroc27fe4c82011-07-25 17:12:10 -0700961 * *Caution*: The caller must hold mmap_sem() if @hugetlb_entry
962 * is used.
Matt Mackalle6473092008-02-04 22:29:01 -0800963 *
964 * (see walk_page_range for more details)
965 */
966struct mm_walk {
Dave Hansen21650092008-06-12 15:21:47 -0700967 int (*pgd_entry)(pgd_t *, unsigned long, unsigned long, struct mm_walk *);
968 int (*pud_entry)(pud_t *, unsigned long, unsigned long, struct mm_walk *);
969 int (*pmd_entry)(pmd_t *, unsigned long, unsigned long, struct mm_walk *);
970 int (*pte_entry)(pte_t *, unsigned long, unsigned long, struct mm_walk *);
971 int (*pte_hole)(unsigned long, unsigned long, struct mm_walk *);
Naoya Horiguchi116354d2010-04-06 14:35:04 -0700972 int (*hugetlb_entry)(pte_t *, unsigned long,
973 unsigned long, unsigned long, struct mm_walk *);
Dave Hansen21650092008-06-12 15:21:47 -0700974 struct mm_struct *mm;
975 void *private;
Matt Mackalle6473092008-02-04 22:29:01 -0800976};
977
Dave Hansen21650092008-06-12 15:21:47 -0700978int walk_page_range(unsigned long addr, unsigned long end,
979 struct mm_walk *walk);
Jan Beulich42b77722008-07-23 21:27:10 -0700980void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
Hugh Dickins3bf5ee92005-04-19 13:29:16 -0700981 unsigned long end, unsigned long floor, unsigned long ceiling);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
983 struct vm_area_struct *vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700984void unmap_mapping_range(struct address_space *mapping,
985 loff_t const holebegin, loff_t const holelen, int even_cows);
Johannes Weiner3b6748e2009-06-16 15:32:35 -0700986int follow_pfn(struct vm_area_struct *vma, unsigned long address,
987 unsigned long *pfn);
venkatesh.pallipadi@intel.comd87fe662008-12-19 13:47:27 -0800988int follow_phys(struct vm_area_struct *vma, unsigned long address,
989 unsigned int flags, unsigned long *prot, resource_size_t *phys);
Rik van Riel28b2ee22008-07-23 21:27:05 -0700990int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
991 void *buf, int len, int write);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700992
993static inline void unmap_shared_mapping_range(struct address_space *mapping,
994 loff_t const holebegin, loff_t const holelen)
995{
996 unmap_mapping_range(mapping, holebegin, holelen, 0);
997}
998
npiggin@suse.de25d9e2d2009-08-21 02:35:05 +1000999extern void truncate_pagecache(struct inode *inode, loff_t old, loff_t new);
Christoph Hellwig2c27c652010-06-04 11:30:04 +02001000extern void truncate_setsize(struct inode *inode, loff_t newsize);
Hugh Dickins623e3db2012-03-28 14:42:40 -07001001void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
Nick Piggin750b4982009-09-16 11:50:12 +02001002int truncate_inode_page(struct address_space *mapping, struct page *page);
Andi Kleen25718732009-09-16 11:50:13 +02001003int generic_error_remove_page(struct address_space *mapping, struct page *page);
Wu Fengguang83f78662009-09-16 11:50:13 +02001004int invalidate_inode_page(struct page *page);
1005
David Howells7ee1dd32006-01-06 00:11:44 -08001006#ifdef CONFIG_MMU
Nick Piggin83c54072007-07-19 01:47:05 -07001007extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
Linus Torvaldsd06063c2009-04-10 09:01:23 -07001008 unsigned long address, unsigned int flags);
Peter Zijlstra5c723ba2011-07-27 12:17:11 +02001009extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
1010 unsigned long address, unsigned int fault_flags);
David Howells7ee1dd32006-01-06 00:11:44 -08001011#else
1012static inline int handle_mm_fault(struct mm_struct *mm,
1013 struct vm_area_struct *vma, unsigned long address,
Linus Torvaldsd06063c2009-04-10 09:01:23 -07001014 unsigned int flags)
David Howells7ee1dd32006-01-06 00:11:44 -08001015{
1016 /* should never happen if there's no MMU */
1017 BUG();
1018 return VM_FAULT_SIGBUS;
1019}
Peter Zijlstra5c723ba2011-07-27 12:17:11 +02001020static inline int fixup_user_fault(struct task_struct *tsk,
1021 struct mm_struct *mm, unsigned long address,
1022 unsigned int fault_flags)
1023{
1024 /* should never happen if there's no MMU */
1025 BUG();
1026 return -EFAULT;
1027}
David Howells7ee1dd32006-01-06 00:11:44 -08001028#endif
Nick Pigginf33ea7f2005-08-03 20:24:01 +10001029
Linus Torvalds1da177e2005-04-16 15:20:36 -07001030extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
Stephen Wilson5ddd36b2011-03-13 15:49:20 -04001031extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
1032 void *buf, int len, int write);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001033
Huang Ying0014bd92011-01-30 11:15:47 +08001034int __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
1035 unsigned long start, int len, unsigned int foll_flags,
1036 struct page **pages, struct vm_area_struct **vmas,
1037 int *nonblocking);
Nick Piggind2bf6be2009-06-16 15:31:39 -07001038int get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
Peter Zijlstra9d737772009-06-25 11:58:55 +02001039 unsigned long start, int nr_pages, int write, int force,
Nick Piggind2bf6be2009-06-16 15:31:39 -07001040 struct page **pages, struct vm_area_struct **vmas);
1041int get_user_pages_fast(unsigned long start, int nr_pages, int write,
1042 struct page **pages);
Mel Gorman18022c52012-07-31 16:44:51 -07001043struct kvec;
1044int get_kernel_pages(const struct kvec *iov, int nr_pages, int write,
1045 struct page **pages);
1046int get_kernel_page(unsigned long start, int write, struct page **pages);
Hugh Dickinsf3e8fcc2009-09-21 17:03:25 -07001047struct page *get_dump_page(unsigned long addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001048
David Howellscf9a2ae2006-08-29 19:05:54 +01001049extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
1050extern void do_invalidatepage(struct page *page, unsigned long offset);
1051
Linus Torvalds1da177e2005-04-16 15:20:36 -07001052int __set_page_dirty_nobuffers(struct page *page);
Ken Chen76719322007-02-10 01:43:15 -08001053int __set_page_dirty_no_writeback(struct page *page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001054int redirty_page_for_writepage(struct writeback_control *wbc,
1055 struct page *page);
Edward Shishkine3a7cca2009-03-31 15:19:39 -07001056void account_page_dirtied(struct page *page, struct address_space *mapping);
Michael Rubinf629d1c2010-10-26 14:21:33 -07001057void account_page_writeback(struct page *page);
Harvey Harrisonb3c97522008-02-13 15:03:15 -08001058int set_page_dirty(struct page *page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001059int set_page_dirty_lock(struct page *page);
1060int clear_page_dirty_for_io(struct page *page);
1061
Stefan Bader39aa3cb2010-08-31 15:52:27 +02001062/* Is the vma a continuation of the stack vma above it? */
Mikulas Patockaa09a79f2011-05-09 13:01:09 +02001063static inline int vma_growsdown(struct vm_area_struct *vma, unsigned long addr)
Stefan Bader39aa3cb2010-08-31 15:52:27 +02001064{
1065 return vma && (vma->vm_end == addr) && (vma->vm_flags & VM_GROWSDOWN);
1066}
1067
Mikulas Patockaa09a79f2011-05-09 13:01:09 +02001068static inline int stack_guard_page_start(struct vm_area_struct *vma,
1069 unsigned long addr)
1070{
1071 return (vma->vm_flags & VM_GROWSDOWN) &&
1072 (vma->vm_start == addr) &&
1073 !vma_growsdown(vma->vm_prev, addr);
1074}
1075
1076/* Is the vma a continuation of the stack vma below it? */
1077static inline int vma_growsup(struct vm_area_struct *vma, unsigned long addr)
1078{
1079 return vma && (vma->vm_start == addr) && (vma->vm_flags & VM_GROWSUP);
1080}
1081
1082static inline int stack_guard_page_end(struct vm_area_struct *vma,
1083 unsigned long addr)
1084{
1085 return (vma->vm_flags & VM_GROWSUP) &&
1086 (vma->vm_end == addr) &&
1087 !vma_growsup(vma->vm_next, addr);
1088}
1089
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -07001090extern pid_t
1091vm_is_stack(struct task_struct *task, struct vm_area_struct *vma, int in_group);
1092
Ollie Wildb6a2fea2007-07-19 01:48:16 -07001093extern unsigned long move_page_tables(struct vm_area_struct *vma,
1094 unsigned long old_addr, struct vm_area_struct *new_vma,
Michel Lespinasse38a76012012-10-08 16:31:50 -07001095 unsigned long new_addr, unsigned long len,
1096 bool need_rmap_locks);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001097extern unsigned long do_mremap(unsigned long addr,
1098 unsigned long old_len, unsigned long new_len,
1099 unsigned long flags, unsigned long new_addr);
Peter Zijlstra7da4d642012-11-19 03:14:23 +01001100extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
1101 unsigned long end, pgprot_t newprot,
Mel Gorman4b10e7d2012-10-25 14:16:32 +02001102 int dirty_accountable, int prot_numa);
Ollie Wildb6a2fea2007-07-19 01:48:16 -07001103extern int mprotect_fixup(struct vm_area_struct *vma,
1104 struct vm_area_struct **pprev, unsigned long start,
1105 unsigned long end, unsigned long newflags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001106
Nick Piggin21cc1992008-07-25 19:45:22 -07001107/*
Peter Zijlstra465a4542009-06-15 12:31:37 +02001108 * doesn't attempt to fault and will return short.
1109 */
1110int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
1111 struct page **pages);
KAMEZAWA Hiroyukid559db02010-03-05 13:41:39 -08001112/*
1113 * per-process(per-mm_struct) statistics.
1114 */
Matt Fleming172703b2011-05-24 17:12:36 -07001115static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
1116{
Konstantin Khlebnikov69c97822012-03-21 16:33:49 -07001117 long val = atomic_long_read(&mm->rss_stat.count[member]);
1118
1119#ifdef SPLIT_RSS_COUNTING
1120 /*
1121 * counter is updated in asynchronous manner and may go to minus.
1122 * But it's never be expected number for users.
1123 */
1124 if (val < 0)
1125 val = 0;
Matt Fleming172703b2011-05-24 17:12:36 -07001126#endif
Konstantin Khlebnikov69c97822012-03-21 16:33:49 -07001127 return (unsigned long)val;
1128}
KAMEZAWA Hiroyukid559db02010-03-05 13:41:39 -08001129
1130static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
1131{
1132 atomic_long_add(value, &mm->rss_stat.count[member]);
1133}
1134
1135static inline void inc_mm_counter(struct mm_struct *mm, int member)
1136{
1137 atomic_long_inc(&mm->rss_stat.count[member]);
1138}
1139
1140static inline void dec_mm_counter(struct mm_struct *mm, int member)
1141{
1142 atomic_long_dec(&mm->rss_stat.count[member]);
1143}
1144
KAMEZAWA Hiroyukid559db02010-03-05 13:41:39 -08001145static inline unsigned long get_mm_rss(struct mm_struct *mm)
1146{
1147 return get_mm_counter(mm, MM_FILEPAGES) +
1148 get_mm_counter(mm, MM_ANONPAGES);
1149}
1150
1151static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
1152{
1153 return max(mm->hiwater_rss, get_mm_rss(mm));
1154}
1155
1156static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
1157{
1158 return max(mm->hiwater_vm, mm->total_vm);
1159}
1160
1161static inline void update_hiwater_rss(struct mm_struct *mm)
1162{
1163 unsigned long _rss = get_mm_rss(mm);
1164
1165 if ((mm)->hiwater_rss < _rss)
1166 (mm)->hiwater_rss = _rss;
1167}
1168
1169static inline void update_hiwater_vm(struct mm_struct *mm)
1170{
1171 if (mm->hiwater_vm < mm->total_vm)
1172 mm->hiwater_vm = mm->total_vm;
1173}
1174
1175static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
1176 struct mm_struct *mm)
1177{
1178 unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
1179
1180 if (*maxrss < hiwater_rss)
1181 *maxrss = hiwater_rss;
1182}
1183
KAMEZAWA Hiroyuki53bddb42010-03-10 15:20:38 -08001184#if defined(SPLIT_RSS_COUNTING)
David Rientjes05af2e12012-03-21 16:34:13 -07001185void sync_mm_rss(struct mm_struct *mm);
KAMEZAWA Hiroyuki53bddb42010-03-10 15:20:38 -08001186#else
David Rientjes05af2e12012-03-21 16:34:13 -07001187static inline void sync_mm_rss(struct mm_struct *mm)
KAMEZAWA Hiroyuki53bddb42010-03-10 15:20:38 -08001188{
1189}
1190#endif
Peter Zijlstra465a4542009-06-15 12:31:37 +02001191
Alexey Dobriyan4e950f62007-07-30 02:36:13 +04001192int vma_wants_writenotify(struct vm_area_struct *vma);
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001193
Namhyung Kim25ca1d62010-10-26 14:21:59 -07001194extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1195 spinlock_t **ptl);
1196static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
1197 spinlock_t **ptl)
1198{
1199 pte_t *ptep;
1200 __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
1201 return ptep;
1202}
Linus Torvaldsc9cfcdd2005-11-29 14:03:14 -08001203
Nick Piggin5f22df02007-05-06 14:49:02 -07001204#ifdef __PAGETABLE_PUD_FOLDED
1205static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
1206 unsigned long address)
1207{
1208 return 0;
1209}
1210#else
Hugh Dickins1bb36302005-10-29 18:16:22 -07001211int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
Nick Piggin5f22df02007-05-06 14:49:02 -07001212#endif
1213
1214#ifdef __PAGETABLE_PMD_FOLDED
1215static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
1216 unsigned long address)
1217{
1218 return 0;
1219}
1220#else
Hugh Dickins1bb36302005-10-29 18:16:22 -07001221int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
Nick Piggin5f22df02007-05-06 14:49:02 -07001222#endif
1223
Andrea Arcangeli8ac1f832011-01-13 15:46:43 -08001224int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
1225 pmd_t *pmd, unsigned long address);
Hugh Dickins1bb36302005-10-29 18:16:22 -07001226int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
1227
Linus Torvalds1da177e2005-04-16 15:20:36 -07001228/*
1229 * The following ifdef needed to get the 4level-fixup.h header to work.
1230 * Remove it when 4level-fixup.h has been removed.
1231 */
Hugh Dickins1bb36302005-10-29 18:16:22 -07001232#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001233static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
1234{
Hugh Dickins1bb36302005-10-29 18:16:22 -07001235 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
1236 NULL: pud_offset(pgd, address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001237}
1238
1239static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
1240{
Hugh Dickins1bb36302005-10-29 18:16:22 -07001241 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
1242 NULL: pmd_offset(pud, address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001243}
Hugh Dickins1bb36302005-10-29 18:16:22 -07001244#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
1245
Jeremy Fitzhardingef7d0b922008-09-09 15:43:22 -07001246#if USE_SPLIT_PTLOCKS
Hugh Dickins4c21e2f2005-10-29 18:16:40 -07001247/*
1248 * We tuck a spinlock to guard each pagetable page into its struct page,
1249 * at page->private, with BUILD_BUG_ON to make sure that this will not
1250 * overflow into the next struct page (as it might with DEBUG_SPINLOCK).
1251 * When freeing, reset page->mapping so free_pages_check won't complain.
1252 */
Andrew Morton349aef02006-01-08 01:04:36 -08001253#define __pte_lockptr(page) &((page)->ptl)
Hugh Dickins4c21e2f2005-10-29 18:16:40 -07001254#define pte_lock_init(_page) do { \
1255 spin_lock_init(__pte_lockptr(_page)); \
1256} while (0)
1257#define pte_lock_deinit(page) ((page)->mapping = NULL)
1258#define pte_lockptr(mm, pmd) ({(void)(mm); __pte_lockptr(pmd_page(*(pmd)));})
Jeremy Fitzhardingef7d0b922008-09-09 15:43:22 -07001259#else /* !USE_SPLIT_PTLOCKS */
Hugh Dickins4c21e2f2005-10-29 18:16:40 -07001260/*
1261 * We use mm->page_table_lock to guard all pagetable pages of the mm.
1262 */
1263#define pte_lock_init(page) do {} while (0)
1264#define pte_lock_deinit(page) do {} while (0)
1265#define pte_lockptr(mm, pmd) ({(void)(pmd); &(mm)->page_table_lock;})
Jeremy Fitzhardingef7d0b922008-09-09 15:43:22 -07001266#endif /* USE_SPLIT_PTLOCKS */
Hugh Dickins4c21e2f2005-10-29 18:16:40 -07001267
Martin Schwidefsky2f569af2008-02-08 04:22:04 -08001268static inline void pgtable_page_ctor(struct page *page)
1269{
1270 pte_lock_init(page);
1271 inc_zone_page_state(page, NR_PAGETABLE);
1272}
1273
1274static inline void pgtable_page_dtor(struct page *page)
1275{
1276 pte_lock_deinit(page);
1277 dec_zone_page_state(page, NR_PAGETABLE);
1278}
1279
Hugh Dickinsc74df322005-10-29 18:16:23 -07001280#define pte_offset_map_lock(mm, pmd, address, ptlp) \
1281({ \
Hugh Dickins4c21e2f2005-10-29 18:16:40 -07001282 spinlock_t *__ptl = pte_lockptr(mm, pmd); \
Hugh Dickinsc74df322005-10-29 18:16:23 -07001283 pte_t *__pte = pte_offset_map(pmd, address); \
1284 *(ptlp) = __ptl; \
1285 spin_lock(__ptl); \
1286 __pte; \
1287})
1288
1289#define pte_unmap_unlock(pte, ptl) do { \
1290 spin_unlock(ptl); \
1291 pte_unmap(pte); \
1292} while (0)
1293
Andrea Arcangeli8ac1f832011-01-13 15:46:43 -08001294#define pte_alloc_map(mm, vma, pmd, address) \
1295 ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, vma, \
1296 pmd, address))? \
1297 NULL: pte_offset_map(pmd, address))
Hugh Dickins1bb36302005-10-29 18:16:22 -07001298
Hugh Dickinsc74df322005-10-29 18:16:23 -07001299#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
Andrea Arcangeli8ac1f832011-01-13 15:46:43 -08001300 ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, NULL, \
1301 pmd, address))? \
Hugh Dickinsc74df322005-10-29 18:16:23 -07001302 NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
1303
Hugh Dickins1bb36302005-10-29 18:16:22 -07001304#define pte_alloc_kernel(pmd, address) \
Andrea Arcangeli8ac1f832011-01-13 15:46:43 -08001305 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
Hugh Dickins1bb36302005-10-29 18:16:22 -07001306 NULL: pte_offset_kernel(pmd, address))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001307
1308extern void free_area_init(unsigned long * zones_size);
Johannes Weiner9109fb72008-07-23 21:27:20 -07001309extern void free_area_init_node(int nid, unsigned long * zones_size,
1310 unsigned long zone_start_pfn, unsigned long *zholes_size);
David Howells49a7f042012-03-28 18:30:03 +01001311extern void free_initmem(void);
1312
Tejun Heo0ee332c2011-12-08 10:22:09 -08001313#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
Mel Gormanc7132162006-09-27 01:49:43 -07001314/*
Tejun Heo0ee332c2011-12-08 10:22:09 -08001315 * With CONFIG_HAVE_MEMBLOCK_NODE_MAP set, an architecture may initialise its
Mel Gormanc7132162006-09-27 01:49:43 -07001316 * zones, allocate the backing mem_map and account for memory holes in a more
1317 * architecture independent manner. This is a substitute for creating the
1318 * zone_sizes[] and zholes_size[] arrays and passing them to
1319 * free_area_init_node()
1320 *
1321 * An architecture is expected to register range of page frames backed by
Tejun Heo0ee332c2011-12-08 10:22:09 -08001322 * physical memory with memblock_add[_node]() before calling
Mel Gormanc7132162006-09-27 01:49:43 -07001323 * free_area_init_nodes() passing in the PFN each zone ends at. At a basic
1324 * usage, an architecture is expected to do something like
1325 *
1326 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
1327 * max_highmem_pfn};
1328 * for_each_valid_physical_page_range()
Tejun Heo0ee332c2011-12-08 10:22:09 -08001329 * memblock_add_node(base, size, nid)
Mel Gormanc7132162006-09-27 01:49:43 -07001330 * free_area_init_nodes(max_zone_pfns);
1331 *
Tejun Heo0ee332c2011-12-08 10:22:09 -08001332 * free_bootmem_with_active_regions() calls free_bootmem_node() for each
1333 * registered physical page range. Similarly
1334 * sparse_memory_present_with_active_regions() calls memory_present() for
1335 * each range when SPARSEMEM is enabled.
Mel Gormanc7132162006-09-27 01:49:43 -07001336 *
1337 * See mm/page_alloc.c for more information on each function exposed by
Tejun Heo0ee332c2011-12-08 10:22:09 -08001338 * CONFIG_HAVE_MEMBLOCK_NODE_MAP.
Mel Gormanc7132162006-09-27 01:49:43 -07001339 */
1340extern void free_area_init_nodes(unsigned long *max_zone_pfn);
Tejun Heo1e019792011-07-12 09:45:34 +02001341unsigned long node_map_pfn_alignment(void);
Yinghai Lu32996252009-12-15 17:59:02 -08001342unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn,
1343 unsigned long end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07001344extern unsigned long absent_pages_in_range(unsigned long start_pfn,
1345 unsigned long end_pfn);
1346extern void get_pfn_range_for_nid(unsigned int nid,
1347 unsigned long *start_pfn, unsigned long *end_pfn);
1348extern unsigned long find_min_pfn_with_active_regions(void);
Mel Gormanc7132162006-09-27 01:49:43 -07001349extern void free_bootmem_with_active_regions(int nid,
1350 unsigned long max_low_pfn);
1351extern void sparse_memory_present_with_active_regions(int nid);
KAMEZAWA Hiroyukif2dbcfa2009-02-18 14:48:32 -08001352
Tang Chen34b71f12013-02-22 16:33:37 -08001353#define MOVABLEMEM_MAP_MAX MAX_NUMNODES
1354struct movablemem_entry {
1355 unsigned long start_pfn; /* start pfn of memory segment */
1356 unsigned long end_pfn; /* end pfn of memory segment (exclusive) */
1357};
1358
1359struct movablemem_map {
Tang Chen01a178a2013-02-22 16:33:49 -08001360 bool acpi; /* true if using SRAT info */
Tang Chen34b71f12013-02-22 16:33:37 -08001361 int nr_map;
1362 struct movablemem_entry map[MOVABLEMEM_MAP_MAX];
Tang Chen27168d32013-02-22 16:33:46 -08001363 nodemask_t numa_nodes_hotplug; /* on which nodes we specify memory */
Tang Chen01a178a2013-02-22 16:33:49 -08001364 nodemask_t numa_nodes_kernel; /* on which nodes kernel resides in */
Tang Chen34b71f12013-02-22 16:33:37 -08001365};
1366
Tang Chen27168d32013-02-22 16:33:46 -08001367extern void __init insert_movablemem_map(unsigned long start_pfn,
1368 unsigned long end_pfn);
1369extern int __init movablemem_map_overlap(unsigned long start_pfn,
1370 unsigned long end_pfn);
Tejun Heo0ee332c2011-12-08 10:22:09 -08001371#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
Tejun Heo5dfe8662011-07-14 09:46:10 +02001372
Tejun Heo0ee332c2011-12-08 10:22:09 -08001373#if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \
KAMEZAWA Hiroyukif2dbcfa2009-02-18 14:48:32 -08001374 !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
1375static inline int __early_pfn_to_nid(unsigned long pfn)
1376{
1377 return 0;
1378}
1379#else
1380/* please see mm/page_alloc.c */
1381extern int __meminit early_pfn_to_nid(unsigned long pfn);
1382#ifdef CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID
1383/* there is a per-arch backend function. */
1384extern int __meminit __early_pfn_to_nid(unsigned long pfn);
1385#endif /* CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID */
1386#endif
1387
Mel Gorman0e0b8642006-09-27 01:49:56 -07001388extern void set_dma_reserve(unsigned long new_dma_reserve);
Dave Hansena2f3aa022007-01-10 23:15:30 -08001389extern void memmap_init_zone(unsigned long, int, unsigned long,
1390 unsigned long, enum memmap_context);
Minchan Kimbc75d332009-06-16 15:32:48 -07001391extern void setup_per_zone_wmarks(void);
KOSAKI Motohiro1b79acc2011-05-24 17:11:32 -07001392extern int __meminit init_per_zone_wmark_min(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001393extern void mem_init(void);
David Howells8feae132009-01-08 12:04:47 +00001394extern void __init mmap_init(void);
David Rientjesb2b755b2011-03-24 15:18:15 -07001395extern void show_mem(unsigned int flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001396extern void si_meminfo(struct sysinfo * val);
1397extern void si_meminfo_node(struct sysinfo *val, int nid);
1398
Joe Perches3ee9a4f2011-10-31 17:08:35 -07001399extern __printf(3, 4)
1400void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...);
Dave Hansena238ab52011-05-24 17:12:16 -07001401
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07001402extern void setup_per_cpu_pageset(void);
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07001403
Shaohua Li112067f2009-09-21 17:01:16 -07001404extern void zone_pcp_update(struct zone *zone);
Jiang Liu340175b2012-07-31 16:43:32 -07001405extern void zone_pcp_reset(struct zone *zone);
Shaohua Li112067f2009-09-21 17:01:16 -07001406
David Howells8feae132009-01-08 12:04:47 +00001407/* nommu.c */
David Howells33e5d7692009-04-02 16:56:32 -07001408extern atomic_long_t mmap_pages_allocated;
David Howells7e660872010-01-15 17:01:39 -08001409extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
David Howells8feae132009-01-08 12:04:47 +00001410
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -07001411/* interval_tree.c */
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -07001412void vma_interval_tree_insert(struct vm_area_struct *node,
1413 struct rb_root *root);
Michel Lespinasse9826a512012-10-08 16:31:35 -07001414void vma_interval_tree_insert_after(struct vm_area_struct *node,
1415 struct vm_area_struct *prev,
1416 struct rb_root *root);
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -07001417void vma_interval_tree_remove(struct vm_area_struct *node,
1418 struct rb_root *root);
1419struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root *root,
1420 unsigned long start, unsigned long last);
1421struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
1422 unsigned long start, unsigned long last);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001423
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -07001424#define vma_interval_tree_foreach(vma, root, start, last) \
1425 for (vma = vma_interval_tree_iter_first(root, start, last); \
1426 vma; vma = vma_interval_tree_iter_next(vma, start, last))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001427
1428static inline void vma_nonlinear_insert(struct vm_area_struct *vma,
1429 struct list_head *list)
1430{
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -07001431 list_add_tail(&vma->shared.nonlinear, list);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001432}
1433
Michel Lespinassebf181b92012-10-08 16:31:39 -07001434void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
1435 struct rb_root *root);
1436void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
1437 struct rb_root *root);
1438struct anon_vma_chain *anon_vma_interval_tree_iter_first(
1439 struct rb_root *root, unsigned long start, unsigned long last);
1440struct anon_vma_chain *anon_vma_interval_tree_iter_next(
1441 struct anon_vma_chain *node, unsigned long start, unsigned long last);
Michel Lespinasseed8ea812012-10-08 16:31:45 -07001442#ifdef CONFIG_DEBUG_VM_RB
1443void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
1444#endif
Michel Lespinassebf181b92012-10-08 16:31:39 -07001445
1446#define anon_vma_interval_tree_foreach(avc, root, start, last) \
1447 for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
1448 avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
1449
Linus Torvalds1da177e2005-04-16 15:20:36 -07001450/* mmap.c */
Alan Cox34b4e4a2007-08-22 14:01:28 -07001451extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
Rik van Riel5beb4932010-03-05 13:42:07 -08001452extern int vma_adjust(struct vm_area_struct *vma, unsigned long start,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001453 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
1454extern struct vm_area_struct *vma_merge(struct mm_struct *,
1455 struct vm_area_struct *prev, unsigned long addr, unsigned long end,
1456 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
1457 struct mempolicy *);
1458extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
1459extern int split_vma(struct mm_struct *,
1460 struct vm_area_struct *, unsigned long addr, int new_below);
1461extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
1462extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
1463 struct rb_node **, struct rb_node *);
Hugh Dickinsa8fb5612005-10-29 18:15:57 -07001464extern void unlink_file_vma(struct vm_area_struct *);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001465extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
Michel Lespinasse38a76012012-10-08 16:31:50 -07001466 unsigned long addr, unsigned long len, pgoff_t pgoff,
1467 bool *need_rmap_locks);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001468extern void exit_mmap(struct mm_struct *);
Matt Helsley925d1c42008-04-29 01:01:36 -07001469
Andrea Arcangeli7906d002008-07-28 15:46:26 -07001470extern int mm_take_all_locks(struct mm_struct *mm);
1471extern void mm_drop_all_locks(struct mm_struct *mm);
1472
Jiri Slaby38646012011-05-26 16:25:46 -07001473extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
1474extern struct file *get_mm_exe_file(struct mm_struct *mm);
Matt Helsley925d1c42008-04-29 01:01:36 -07001475
akpm@osdl.org119f6572005-05-01 08:58:35 -07001476extern int may_expand_vm(struct mm_struct *mm, unsigned long npages);
Roland McGrathfa5dc222007-02-08 14:20:41 -08001477extern int install_special_mapping(struct mm_struct *mm,
1478 unsigned long addr, unsigned long len,
1479 unsigned long flags, struct page **pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001480
1481extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
1482
Miklos Szeredi0165ab42007-07-15 23:38:26 -07001483extern unsigned long mmap_region(struct file *file, unsigned long addr,
Michel Lespinassec22c0d62013-02-22 16:32:43 -08001484 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff);
Michel Lespinassebebeb3d2013-02-22 16:32:37 -08001485extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
1486 unsigned long len, unsigned long prot, unsigned long flags,
Michel Lespinasse41badc12013-02-22 16:32:47 -08001487 unsigned long pgoff, unsigned long *populate);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001488extern int do_munmap(struct mm_struct *, unsigned long, size_t);
1489
Michel Lespinassebebeb3d2013-02-22 16:32:37 -08001490#ifdef CONFIG_MMU
1491extern int __mm_populate(unsigned long addr, unsigned long len,
1492 int ignore_errors);
1493static inline void mm_populate(unsigned long addr, unsigned long len)
1494{
1495 /* Ignore errors */
1496 (void) __mm_populate(addr, len, 1);
1497}
1498#else
1499static inline void mm_populate(unsigned long addr, unsigned long len) {}
1500#endif
1501
Linus Torvaldse4eb1ff2012-04-20 15:35:40 -07001502/* These take the mm semaphore themselves */
1503extern unsigned long vm_brk(unsigned long, unsigned long);
Al Virobfce2812012-04-20 21:57:04 -04001504extern int vm_munmap(unsigned long, size_t);
Linus Torvalds6be5ceb2012-04-20 17:13:58 -07001505extern unsigned long vm_mmap(struct file *, unsigned long,
1506 unsigned long, unsigned long,
1507 unsigned long, unsigned long);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001508
Michel Lespinassedb4fbfb2012-12-11 16:01:49 -08001509struct vm_unmapped_area_info {
1510#define VM_UNMAPPED_AREA_TOPDOWN 1
1511 unsigned long flags;
1512 unsigned long length;
1513 unsigned long low_limit;
1514 unsigned long high_limit;
1515 unsigned long align_mask;
1516 unsigned long align_offset;
1517};
1518
1519extern unsigned long unmapped_area(struct vm_unmapped_area_info *info);
1520extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
1521
1522/*
1523 * Search for an unmapped address range.
1524 *
1525 * We are looking for a range that:
1526 * - does not intersect with any VMA;
1527 * - is contained within the [low_limit, high_limit) interval;
1528 * - is at least the desired size.
1529 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
1530 */
1531static inline unsigned long
1532vm_unmapped_area(struct vm_unmapped_area_info *info)
1533{
1534 if (!(info->flags & VM_UNMAPPED_AREA_TOPDOWN))
1535 return unmapped_area(info);
1536 else
1537 return unmapped_area_topdown(info);
1538}
1539
Hugh Dickins85821aa2011-07-25 17:12:23 -07001540/* truncate.c */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001541extern void truncate_inode_pages(struct address_space *, loff_t);
Hans Reiserd7339072006-01-06 00:10:36 -08001542extern void truncate_inode_pages_range(struct address_space *,
1543 loff_t lstart, loff_t lend);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001544
1545/* generic vm_area_ops exported for stackable file systems */
Nick Piggind0217ac2007-07-19 01:47:03 -07001546extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
Jan Kara4fcf1c62012-06-12 16:20:29 +02001547extern int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001548
1549/* mm/page-writeback.c */
1550int write_one_page(struct page *page, int wait);
Nick Piggin1cf6e7d2009-02-18 14:48:18 -08001551void task_dirty_inc(struct task_struct *tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001552
1553/* readahead.c */
1554#define VM_MAX_READAHEAD 128 /* kbytes */
1555#define VM_MIN_READAHEAD 16 /* kbytes (includes current page) */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001556
Linus Torvalds1da177e2005-04-16 15:20:36 -07001557int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
Andrew Morton7361f4d2005-11-07 00:59:28 -08001558 pgoff_t offset, unsigned long nr_to_read);
Rusty Russellcf914a72007-07-19 01:48:08 -07001559
1560void page_cache_sync_readahead(struct address_space *mapping,
1561 struct file_ra_state *ra,
1562 struct file *filp,
1563 pgoff_t offset,
1564 unsigned long size);
1565
1566void page_cache_async_readahead(struct address_space *mapping,
1567 struct file_ra_state *ra,
1568 struct file *filp,
1569 struct page *pg,
1570 pgoff_t offset,
1571 unsigned long size);
1572
Linus Torvalds1da177e2005-04-16 15:20:36 -07001573unsigned long max_sane_readahead(unsigned long nr);
Wu Fengguangd30a1102009-06-16 15:31:30 -07001574unsigned long ra_submit(struct file_ra_state *ra,
1575 struct address_space *mapping,
1576 struct file *filp);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001577
Michal Hockod05f3162011-05-24 17:11:44 -07001578/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
Hugh Dickins46dea3d2005-10-29 18:16:20 -07001579extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
Michal Hockod05f3162011-05-24 17:11:44 -07001580
1581/* CONFIG_STACK_GROWSUP still needs to to grow downwards at some places */
1582extern int expand_downwards(struct vm_area_struct *vma,
1583 unsigned long address);
Luck, Tony8ca3eb082010-08-24 11:44:18 -07001584#if VM_GROWSUP
Hugh Dickins46dea3d2005-10-29 18:16:20 -07001585extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
Luck, Tony8ca3eb082010-08-24 11:44:18 -07001586#else
1587 #define expand_upwards(vma, address) do { } while (0)
Matthew Wilcox9ab88512005-11-18 16:16:42 -05001588#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001589
1590/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
1591extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
1592extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
1593 struct vm_area_struct **pprev);
1594
1595/* Look up the first VMA which intersects the interval start_addr..end_addr-1,
1596 NULL if none. Assume start_addr < end_addr. */
1597static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
1598{
1599 struct vm_area_struct * vma = find_vma(mm,start_addr);
1600
1601 if (vma && end_addr <= vma->vm_start)
1602 vma = NULL;
1603 return vma;
1604}
1605
1606static inline unsigned long vma_pages(struct vm_area_struct *vma)
1607{
1608 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
1609}
1610
Pavel Emelyanov640708a2012-01-10 15:11:23 -08001611/* Look up the first VMA which exactly match the interval vm_start ... vm_end */
1612static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
1613 unsigned long vm_start, unsigned long vm_end)
1614{
1615 struct vm_area_struct *vma = find_vma(mm, vm_start);
1616
1617 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
1618 vma = NULL;
1619
1620 return vma;
1621}
1622
David Howellsbad849b2010-08-26 16:00:34 +01001623#ifdef CONFIG_MMU
Hugh Dickins804af2c2006-07-26 21:39:49 +01001624pgprot_t vm_get_page_prot(unsigned long vm_flags);
David Howellsbad849b2010-08-26 16:00:34 +01001625#else
1626static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
1627{
1628 return __pgprot(0);
1629}
1630#endif
1631
Lee Schermerhornb24f53a2012-10-25 14:16:32 +02001632#ifdef CONFIG_ARCH_USES_NUMA_PROT_NONE
Mel Gorman4b10e7d2012-10-25 14:16:32 +02001633unsigned long change_prot_numa(struct vm_area_struct *vma,
Lee Schermerhornb24f53a2012-10-25 14:16:32 +02001634 unsigned long start, unsigned long end);
1635#endif
1636
Hugh Dickinsdeceb6c2005-10-29 18:16:33 -07001637struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
Hugh Dickinsdeceb6c2005-10-29 18:16:33 -07001638int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
1639 unsigned long pfn, unsigned long size, pgprot_t);
Linus Torvaldsa145dd42005-11-30 09:35:19 -08001640int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
Nick Piggine0dc0d82007-02-12 00:51:36 -08001641int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
1642 unsigned long pfn);
Nick Piggin423bad602008-04-28 02:13:01 -07001643int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
1644 unsigned long pfn);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001645
Linus Torvalds6aab3412005-11-28 14:34:23 -08001646struct page *follow_page(struct vm_area_struct *, unsigned long address,
Hugh Dickinsdeceb6c2005-10-29 18:16:33 -07001647 unsigned int foll_flags);
1648#define FOLL_WRITE 0x01 /* check pte is writable */
1649#define FOLL_TOUCH 0x02 /* mark page accessed */
1650#define FOLL_GET 0x04 /* do get_page on page */
Hugh Dickins8e4b9a62009-09-21 17:03:26 -07001651#define FOLL_DUMP 0x08 /* give error on hole if it would be zero */
Hugh Dickins58fa8792009-09-21 17:03:31 -07001652#define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */
Gleb Natapov318b2752011-03-22 16:30:51 -07001653#define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO
1654 * and return without waiting upon it */
Michel Lespinasse110d74a92011-01-13 15:46:11 -08001655#define FOLL_MLOCK 0x40 /* mark page as mlocked */
Andrea Arcangeli500d65d2011-01-13 15:46:55 -08001656#define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */
Huang Ying69ebb832011-01-30 11:15:48 +08001657#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
Andrea Arcangeli0b9d7052012-10-05 21:36:27 +02001658#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001659
Martin Schwidefsky2f569af2008-02-08 04:22:04 -08001660typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
Jeremy Fitzhardingeaee16b32007-05-06 14:48:54 -07001661 void *data);
1662extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
1663 unsigned long size, pte_fn_t fn, void *data);
1664
Linus Torvalds1da177e2005-04-16 15:20:36 -07001665#ifdef CONFIG_PROC_FS
Hugh Dickinsab50b8e2005-10-29 18:15:56 -07001666void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001667#else
Hugh Dickinsab50b8e2005-10-29 18:15:56 -07001668static inline void vm_stat_account(struct mm_struct *mm,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001669 unsigned long flags, struct file *file, long pages)
1670{
Huang Shijie44de9d02012-07-31 16:41:49 -07001671 mm->total_vm += pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001672}
1673#endif /* CONFIG_PROC_FS */
1674
Ingo Molnar12d6f212008-01-30 13:33:58 +01001675#ifdef CONFIG_DEBUG_PAGEALLOC
Ingo Molnar12d6f212008-01-30 13:33:58 +01001676extern void kernel_map_pages(struct page *page, int numpages, int enable);
Rafael J. Wysocki8a235ef2008-02-20 01:47:44 +01001677#ifdef CONFIG_HIBERNATION
1678extern bool kernel_page_present(struct page *page);
1679#endif /* CONFIG_HIBERNATION */
Ingo Molnar12d6f212008-01-30 13:33:58 +01001680#else
Linus Torvalds1da177e2005-04-16 15:20:36 -07001681static inline void
Nick Piggin9858db52006-10-11 01:21:30 -07001682kernel_map_pages(struct page *page, int numpages, int enable) {}
Rafael J. Wysocki8a235ef2008-02-20 01:47:44 +01001683#ifdef CONFIG_HIBERNATION
1684static inline bool kernel_page_present(struct page *page) { return true; }
1685#endif /* CONFIG_HIBERNATION */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001686#endif
1687
Stephen Wilson31db58b2011-03-13 15:49:15 -04001688extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001689#ifdef __HAVE_ARCH_GATE_AREA
Stephen Wilsoncae5d392011-03-13 15:49:17 -04001690int in_gate_area_no_mm(unsigned long addr);
Stephen Wilson83b964b2011-03-13 15:49:16 -04001691int in_gate_area(struct mm_struct *mm, unsigned long addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001692#else
Stephen Wilsoncae5d392011-03-13 15:49:17 -04001693int in_gate_area_no_mm(unsigned long addr);
1694#define in_gate_area(mm, addr) ({(void)mm; in_gate_area_no_mm(addr);})
Linus Torvalds1da177e2005-04-16 15:20:36 -07001695#endif /* __HAVE_ARCH_GATE_AREA */
1696
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07001697int drop_caches_sysctl_handler(struct ctl_table *, int,
Andrew Morton9d0243b2006-01-08 01:00:39 -08001698 void __user *, size_t *, loff_t *);
Ying Hana09ed5e2011-05-24 17:12:26 -07001699unsigned long shrink_slab(struct shrink_control *shrink,
Ying Han1495f232011-05-24 17:12:27 -07001700 unsigned long nr_pages_scanned,
1701 unsigned long lru_pages);
Andrew Morton9d0243b2006-01-08 01:00:39 -08001702
Luke Yang7a9166e2006-02-20 18:28:07 -08001703#ifndef CONFIG_MMU
1704#define randomize_va_space 0
1705#else
Andi Kleena62eaf12006-02-16 23:41:58 +01001706extern int randomize_va_space;
Luke Yang7a9166e2006-02-20 18:28:07 -08001707#endif
Andi Kleena62eaf12006-02-16 23:41:58 +01001708
Sam Ravnborg045e72a2007-07-26 10:41:13 -07001709const char * arch_vma_name(struct vm_area_struct *vma);
Andi Kleen03252912008-01-30 13:33:18 +01001710void print_vma_addr(char *prefix, unsigned long rip);
Ingo Molnare6e54942006-06-27 02:53:50 -07001711
Yinghai Lu9bdac912010-02-10 01:20:22 -08001712void sparse_mem_maps_populate_node(struct page **map_map,
1713 unsigned long pnum_begin,
1714 unsigned long pnum_end,
1715 unsigned long map_count,
1716 int nodeid);
1717
Yasunori Goto98f3cfc2007-10-16 01:26:14 -07001718struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
Andy Whitcroft29c71112007-10-16 01:24:14 -07001719pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
1720pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
1721pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
1722pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
Christoph Lameter8f6aac42007-10-16 01:24:13 -07001723void *vmemmap_alloc_block(unsigned long size, int node);
Yinghai Lu9bdac912010-02-10 01:20:22 -08001724void *vmemmap_alloc_block_buf(unsigned long size, int node);
Christoph Lameter8f6aac42007-10-16 01:24:13 -07001725void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
Andy Whitcroft29c71112007-10-16 01:24:14 -07001726int vmemmap_populate_basepages(struct page *start_page,
1727 unsigned long pages, int node);
1728int vmemmap_populate(struct page *start_page, unsigned long pages, int node);
Yinghai Luc2b91e22008-04-12 01:19:24 -07001729void vmemmap_populate_print_last(void);
Tang Chen01975182013-02-22 16:33:08 -08001730#ifdef CONFIG_MEMORY_HOTPLUG
1731void vmemmap_free(struct page *memmap, unsigned long nr_pages);
1732#endif
Yasuaki Ishimatsu46723bf2013-02-22 16:33:00 -08001733void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
1734 unsigned long size);
Andi Kleen6a460792009-09-16 11:50:15 +02001735
Andi Kleen82ba0112009-12-16 12:19:57 +01001736enum mf_flags {
1737 MF_COUNT_INCREASED = 1 << 0,
Tony Luck7329bbe2011-12-13 09:27:58 -08001738 MF_ACTION_REQUIRED = 1 << 1,
Tony Luck6751ed62012-07-11 10:20:47 -07001739 MF_MUST_KILL = 1 << 2,
Andi Kleen82ba0112009-12-16 12:19:57 +01001740};
Tony Luckcd42f4a2011-12-15 10:48:12 -08001741extern int memory_failure(unsigned long pfn, int trapno, int flags);
Huang Yingea8f5fb2011-07-13 13:14:27 +08001742extern void memory_failure_queue(unsigned long pfn, int trapno, int flags);
Wu Fengguang847ce402009-12-16 12:19:58 +01001743extern int unpoison_memory(unsigned long pfn);
Andi Kleen6a460792009-09-16 11:50:15 +02001744extern int sysctl_memory_failure_early_kill;
1745extern int sysctl_memory_failure_recovery;
Andi Kleenfacb6012009-12-16 12:20:00 +01001746extern void shake_page(struct page *p, int access);
Xishi Qiu293c07e2013-02-22 16:34:02 -08001747extern atomic_long_t num_poisoned_pages;
Andi Kleenfacb6012009-12-16 12:20:00 +01001748extern int soft_offline_page(struct page *page, int flags);
Andi Kleen6a460792009-09-16 11:50:15 +02001749
Wu Fengguang718a3822010-03-10 15:20:43 -08001750extern void dump_page(struct page *page);
1751
Andrea Arcangeli47ad8472011-01-13 15:46:47 -08001752#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
1753extern void clear_huge_page(struct page *page,
1754 unsigned long addr,
1755 unsigned int pages_per_huge_page);
1756extern void copy_user_huge_page(struct page *dst, struct page *src,
1757 unsigned long addr, struct vm_area_struct *vma,
1758 unsigned int pages_per_huge_page);
1759#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
1760
Stanislaw Gruszkac0a32fc2012-01-10 15:07:28 -08001761#ifdef CONFIG_DEBUG_PAGEALLOC
1762extern unsigned int _debug_guardpage_minorder;
1763
1764static inline unsigned int debug_guardpage_minorder(void)
1765{
1766 return _debug_guardpage_minorder;
1767}
1768
1769static inline bool page_is_guard(struct page *page)
1770{
1771 return test_bit(PAGE_DEBUG_FLAG_GUARD, &page->debug_flags);
1772}
1773#else
1774static inline unsigned int debug_guardpage_minorder(void) { return 0; }
1775static inline bool page_is_guard(struct page *page) { return false; }
1776#endif /* CONFIG_DEBUG_PAGEALLOC */
1777
Linus Torvalds1da177e2005-04-16 15:20:36 -07001778#endif /* __KERNEL__ */
1779#endif /* _LINUX_MM_H */