blob: 0755b9fd03a7d936e805efb71cfc1d371ddea0d6 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef _LINUX_MM_H
2#define _LINUX_MM_H
3
Linus Torvalds1da177e2005-04-16 15:20:36 -07004#include <linux/errno.h>
5
6#ifdef __KERNEL__
7
Sasha Levin309381fea2014-01-23 15:52:54 -08008#include <linux/mmdebug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -07009#include <linux/gfp.h>
Paul Gortmaker187f1882011-11-23 20:12:59 -050010#include <linux/bug.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070011#include <linux/list.h>
12#include <linux/mmzone.h>
13#include <linux/rbtree.h>
Konstantin Khlebnikov83aeead2011-12-08 14:33:54 -080014#include <linux/atomic.h>
Ingo Molnar9a11b49a2006-07-03 00:24:33 -070015#include <linux/debug_locks.h>
Heiko Carstens5b99cd02006-09-27 01:50:01 -070016#include <linux/mm_types.h>
Yinghai Lu08677212010-02-10 01:20:20 -080017#include <linux/range.h>
Chris Metcalfc6f6b592010-05-24 14:32:53 -070018#include <linux/pfn.h>
Andrea Arcangelie9da73d2011-01-13 15:46:32 -080019#include <linux/bit_spinlock.h>
Dave Chinnerb0d40c92011-07-08 14:14:42 +100020#include <linux/shrinker.h>
Cyrill Gorcunov9c599022014-10-09 15:27:29 -070021#include <linux/resource.h>
Joonsoo Kime30825f2014-12-12 16:55:49 -080022#include <linux/page_ext.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070023
24struct mempolicy;
25struct anon_vma;
Michel Lespinassebf181b92012-10-08 16:31:39 -070026struct anon_vma_chain;
Alexey Dobriyan4e950f62007-07-30 02:36:13 +040027struct file_ra_state;
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +040028struct user_struct;
Alexey Dobriyan4e950f62007-07-30 02:36:13 +040029struct writeback_control;
Linus Torvalds1da177e2005-04-16 15:20:36 -070030
Jiang Liufccc9982013-07-03 15:04:23 -070031#ifndef CONFIG_NEED_MULTIPLE_NODES /* Don't use mapnrs, do it properly */
Linus Torvalds1da177e2005-04-16 15:20:36 -070032extern unsigned long max_mapnr;
Jiang Liufccc9982013-07-03 15:04:23 -070033
34static inline void set_max_mapnr(unsigned long limit)
35{
36 max_mapnr = limit;
37}
38#else
39static inline void set_max_mapnr(unsigned long limit) { }
Linus Torvalds1da177e2005-04-16 15:20:36 -070040#endif
41
Jan Beulich44813742009-09-21 17:03:05 -070042extern unsigned long totalram_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -070043extern void * high_memory;
Linus Torvalds1da177e2005-04-16 15:20:36 -070044extern int page_cluster;
45
46#ifdef CONFIG_SYSCTL
47extern int sysctl_legacy_va_layout;
48#else
49#define sysctl_legacy_va_layout 0
50#endif
51
52#include <asm/page.h>
53#include <asm/pgtable.h>
54#include <asm/processor.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070055
Tang Chen79442ed2013-11-12 15:07:59 -080056#ifndef __pa_symbol
57#define __pa_symbol(x) __pa(RELOC_HIDE((unsigned long)(x), 0))
58#endif
59
Dominik Dingel593befa2014-10-23 12:07:44 +020060/*
61 * To prevent common memory management code establishing
62 * a zero page mapping on a read fault.
63 * This macro should be defined within <asm/pgtable.h>.
64 * s390 does this to prevent multiplexing of hardware bits
65 * related to the physical page in case of virtualization.
66 */
67#ifndef mm_forbids_zeropage
68#define mm_forbids_zeropage(X) (0)
69#endif
70
Andrew Shewmakerc9b1d092013-04-29 15:08:10 -070071extern unsigned long sysctl_user_reserve_kbytes;
Andrew Shewmaker4eeab4f2013-04-29 15:08:11 -070072extern unsigned long sysctl_admin_reserve_kbytes;
Andrew Shewmakerc9b1d092013-04-29 15:08:10 -070073
Jerome Marchand49f0ce52014-01-21 15:49:14 -080074extern int sysctl_overcommit_memory;
75extern int sysctl_overcommit_ratio;
76extern unsigned long sysctl_overcommit_kbytes;
77
78extern int overcommit_ratio_handler(struct ctl_table *, int, void __user *,
79 size_t *, loff_t *);
80extern int overcommit_kbytes_handler(struct ctl_table *, int, void __user *,
81 size_t *, loff_t *);
82
Linus Torvalds1da177e2005-04-16 15:20:36 -070083#define nth_page(page,n) pfn_to_page(page_to_pfn((page)) + (n))
84
Andrea Righi27ac7922008-07-23 21:28:13 -070085/* to align the pointer to the (next) page boundary */
86#define PAGE_ALIGN(addr) ALIGN(addr, PAGE_SIZE)
87
Andrew Morton0fa73b82013-07-03 15:02:11 -070088/* test whether an address (unsigned long or pointer) is aligned to PAGE_SIZE */
89#define PAGE_ALIGNED(addr) IS_ALIGNED((unsigned long)addr, PAGE_SIZE)
90
Linus Torvalds1da177e2005-04-16 15:20:36 -070091/*
92 * Linux kernel virtual memory manager primitives.
93 * The idea being to have a "virtual" mm in the same way
94 * we have a virtual fs - giving a cleaner interface to the
95 * mm details, and allowing different kinds of memory mappings
96 * (from shared memory to executable loading to arbitrary
97 * mmap() functions).
98 */
99
Christoph Lameterc43692e2006-12-06 20:32:48 -0800100extern struct kmem_cache *vm_area_cachep;
101
Linus Torvalds1da177e2005-04-16 15:20:36 -0700102#ifndef CONFIG_MMU
David Howells8feae132009-01-08 12:04:47 +0000103extern struct rb_root nommu_region_tree;
104extern struct rw_semaphore nommu_region_sem;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105
106extern unsigned int kobjsize(const void *objp);
107#endif
108
109/*
Hugh Dickins605d9282008-08-16 11:07:21 +0100110 * vm_flags in vm_area_struct, see mm_types.h.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111 */
Konstantin Khlebnikovcc2383e2012-10-08 16:28:37 -0700112#define VM_NONE 0x00000000
113
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114#define VM_READ 0x00000001 /* currently active flags */
115#define VM_WRITE 0x00000002
116#define VM_EXEC 0x00000004
117#define VM_SHARED 0x00000008
118
Paolo 'Blaisorblade' Giarrusso7e2cff42005-09-21 09:55:39 -0700119/* mprotect() hardcodes VM_MAYREAD >> 4 == VM_READ, and so for r/w/x bits. */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700120#define VM_MAYREAD 0x00000010 /* limits for mprotect() etc */
121#define VM_MAYWRITE 0x00000020
122#define VM_MAYEXEC 0x00000040
123#define VM_MAYSHARE 0x00000080
124
125#define VM_GROWSDOWN 0x00000100 /* general info on the segment */
Linus Torvalds6aab3412005-11-28 14:34:23 -0800126#define VM_PFNMAP 0x00000400 /* Page-ranges managed without "struct page", just pure PFN */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700127#define VM_DENYWRITE 0x00000800 /* ETXTBSY on write attempts.. */
128
Linus Torvalds1da177e2005-04-16 15:20:36 -0700129#define VM_LOCKED 0x00002000
130#define VM_IO 0x00004000 /* Memory mapped I/O or similar */
131
132 /* Used by sys_madvise() */
133#define VM_SEQ_READ 0x00008000 /* App will access data sequentially */
134#define VM_RAND_READ 0x00010000 /* App will not benefit from clustered reads */
135
136#define VM_DONTCOPY 0x00020000 /* Do not copy this vma on fork */
137#define VM_DONTEXPAND 0x00040000 /* Cannot expand with mremap() */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700138#define VM_ACCOUNT 0x00100000 /* Is a VM accounted object */
Andy Whitcroftcdfd4322008-07-23 21:27:28 -0700139#define VM_NORESERVE 0x00200000 /* should the VM suppress accounting */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700140#define VM_HUGETLB 0x00400000 /* Huge TLB Page VM */
Konstantin Khlebnikovcc2383e2012-10-08 16:28:37 -0700141#define VM_ARCH_1 0x01000000 /* Architecture-specific flag */
Qiaowei Ren4aae7e42014-11-14 07:18:25 -0800142#define VM_ARCH_2 0x02000000
Konstantin Khlebnikov0103bd12012-10-08 16:28:59 -0700143#define VM_DONTDUMP 0x04000000 /* Do not include in the core dump */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144
Cyrill Gorcunovd9104d12013-09-11 14:22:24 -0700145#ifdef CONFIG_MEM_SOFT_DIRTY
146# define VM_SOFTDIRTY 0x08000000 /* Not soft dirty clean area */
147#else
148# define VM_SOFTDIRTY 0
149#endif
150
Jared Hulbertb379d792008-04-28 02:12:58 -0700151#define VM_MIXEDMAP 0x10000000 /* Can contain "struct page" and pure PFN pages */
Konstantin Khlebnikovcc2383e2012-10-08 16:28:37 -0700152#define VM_HUGEPAGE 0x20000000 /* MADV_HUGEPAGE marked this vma */
153#define VM_NOHUGEPAGE 0x40000000 /* MADV_NOHUGEPAGE marked this vma */
Hugh Dickinsf8af4da2009-09-21 17:01:57 -0700154#define VM_MERGEABLE 0x80000000 /* KSM may merge identical pages */
Nick Piggind00806b2007-07-19 01:46:57 -0700155
Konstantin Khlebnikovcc2383e2012-10-08 16:28:37 -0700156#if defined(CONFIG_X86)
157# define VM_PAT VM_ARCH_1 /* PAT reserves whole VMA at once (x86) */
158#elif defined(CONFIG_PPC)
159# define VM_SAO VM_ARCH_1 /* Strong Access Ordering (powerpc) */
160#elif defined(CONFIG_PARISC)
161# define VM_GROWSUP VM_ARCH_1
James Hogan9ca52ed2012-10-16 10:16:14 +0100162#elif defined(CONFIG_METAG)
163# define VM_GROWSUP VM_ARCH_1
Konstantin Khlebnikovcc2383e2012-10-08 16:28:37 -0700164#elif defined(CONFIG_IA64)
165# define VM_GROWSUP VM_ARCH_1
166#elif !defined(CONFIG_MMU)
167# define VM_MAPPED_COPY VM_ARCH_1 /* T if mapped copy of data (nommu mmap) */
168#endif
169
Qiaowei Ren4aae7e42014-11-14 07:18:25 -0800170#if defined(CONFIG_X86)
171/* MPX specific bounds table or bounds directory */
172# define VM_MPX VM_ARCH_2
173#endif
174
Konstantin Khlebnikovcc2383e2012-10-08 16:28:37 -0700175#ifndef VM_GROWSUP
176# define VM_GROWSUP VM_NONE
177#endif
178
Mel Gormana8bef8ff2010-05-24 14:32:24 -0700179/* Bits set in the VMA until the stack is in its final location */
180#define VM_STACK_INCOMPLETE_SETUP (VM_RAND_READ | VM_SEQ_READ)
181
Linus Torvalds1da177e2005-04-16 15:20:36 -0700182#ifndef VM_STACK_DEFAULT_FLAGS /* arch can override this */
183#define VM_STACK_DEFAULT_FLAGS VM_DATA_DEFAULT_FLAGS
184#endif
185
186#ifdef CONFIG_STACK_GROWSUP
187#define VM_STACK_FLAGS (VM_GROWSUP | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
188#else
189#define VM_STACK_FLAGS (VM_GROWSDOWN | VM_STACK_DEFAULT_FLAGS | VM_ACCOUNT)
190#endif
191
Linus Torvalds1da177e2005-04-16 15:20:36 -0700192/*
Andrea Arcangeli78f11a22011-04-27 15:26:45 -0700193 * Special vmas that are non-mergable, non-mlock()able.
194 * Note: mm/huge_memory.c VM_NO_THP depends on this definition.
Nick Pigginb291f002008-10-18 20:26:44 -0700195 */
Vlastimil Babka9050d7e2014-03-03 15:38:27 -0800196#define VM_SPECIAL (VM_IO | VM_DONTEXPAND | VM_PFNMAP | VM_MIXEDMAP)
Nick Pigginb291f002008-10-18 20:26:44 -0700197
Alex Thorltona0715cc2014-04-07 15:37:10 -0700198/* This mask defines which mm->def_flags a process can inherit its parent */
199#define VM_INIT_DEF_MASK VM_NOHUGEPAGE
200
Nick Pigginb291f002008-10-18 20:26:44 -0700201/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700202 * mapping from the currently active vm_flags protection bits (the
203 * low four bits) to a page protection mask..
204 */
205extern pgprot_t protection_map[16];
206
Nick Piggind0217ac2007-07-19 01:47:03 -0700207#define FAULT_FLAG_WRITE 0x01 /* Fault was a write access */
Kirill A. Shutemov9b4bdd22015-02-10 14:09:51 -0800208#define FAULT_FLAG_MKWRITE 0x02 /* Fault was mkwrite of existing pte */
209#define FAULT_FLAG_ALLOW_RETRY 0x04 /* Retry fault if blocking */
210#define FAULT_FLAG_RETRY_NOWAIT 0x08 /* Don't drop mmap_sem and wait when retrying */
211#define FAULT_FLAG_KILLABLE 0x10 /* The fault task is in SIGKILL killable region */
212#define FAULT_FLAG_TRIED 0x20 /* Second try */
213#define FAULT_FLAG_USER 0x40 /* The fault originated in userspace */
Nick Piggind0217ac2007-07-19 01:47:03 -0700214
venkatesh.pallipadi@intel.com6bd9cd52008-12-19 13:47:26 -0800215/*
Nick Piggind0217ac2007-07-19 01:47:03 -0700216 * vm_fault is filled by the the pagefault handler and passed to the vma's
Nick Piggin83c54072007-07-19 01:47:05 -0700217 * ->fault function. The vma's ->fault is responsible for returning a bitmask
218 * of VM_FAULT_xxx flags that give details about how the fault was handled.
Nick Piggin54cb8822007-07-19 01:46:59 -0700219 *
Kirill A. Shutemov9b4bdd22015-02-10 14:09:51 -0800220 * pgoff should be used in favour of virtual_address, if possible.
Nick Piggin54cb8822007-07-19 01:46:59 -0700221 */
Nick Piggind0217ac2007-07-19 01:47:03 -0700222struct vm_fault {
223 unsigned int flags; /* FAULT_FLAG_xxx flags */
224 pgoff_t pgoff; /* Logical page offset based on vma */
225 void __user *virtual_address; /* Faulting virtual address */
226
Matthew Wilcox2e4cdab2015-02-16 15:58:50 -0800227 struct page *cow_page; /* Handler may choose to COW */
Nick Piggind0217ac2007-07-19 01:47:03 -0700228 struct page *page; /* ->fault handlers should return a
Nick Piggin83c54072007-07-19 01:47:05 -0700229 * page here, unless VM_FAULT_NOPAGE
Nick Piggind0217ac2007-07-19 01:47:03 -0700230 * is set (which is also implied by
Nick Piggin83c54072007-07-19 01:47:05 -0700231 * VM_FAULT_ERROR).
Nick Piggind0217ac2007-07-19 01:47:03 -0700232 */
Kirill A. Shutemov8c6e50b2014-04-07 15:37:18 -0700233 /* for ->map_pages() only */
234 pgoff_t max_pgoff; /* map pages for offset from pgoff till
235 * max_pgoff inclusive */
236 pte_t *pte; /* pte entry associated with ->pgoff */
Nick Piggin54cb8822007-07-19 01:46:59 -0700237};
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238
239/*
240 * These are the virtual MM functions - opening of an area, closing and
241 * unmapping it (needed to keep files on disk up-to-date etc), pointer
242 * to the functions called when a no-page or a wp-page exception occurs.
243 */
244struct vm_operations_struct {
245 void (*open)(struct vm_area_struct * area);
246 void (*close)(struct vm_area_struct * area);
Nick Piggind0217ac2007-07-19 01:47:03 -0700247 int (*fault)(struct vm_area_struct *vma, struct vm_fault *vmf);
Kirill A. Shutemov8c6e50b2014-04-07 15:37:18 -0700248 void (*map_pages)(struct vm_area_struct *vma, struct vm_fault *vmf);
David Howells9637a5e2006-06-23 02:03:43 -0700249
250 /* notification that a previously read-only page is about to become
251 * writable, if an error is returned it will cause a SIGBUS */
Nick Pigginc2ec1752009-03-31 15:23:21 -0700252 int (*page_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
Rik van Riel28b2ee22008-07-23 21:27:05 -0700253
Boaz Harroshdd906182015-04-15 16:15:11 -0700254 /* same as page_mkwrite when using VM_PFNMAP|VM_MIXEDMAP */
255 int (*pfn_mkwrite)(struct vm_area_struct *vma, struct vm_fault *vmf);
256
Rik van Riel28b2ee22008-07-23 21:27:05 -0700257 /* called by access_process_vm when get_user_pages() fails, typically
258 * for use by special VMAs that can switch between memory and hardware
259 */
260 int (*access)(struct vm_area_struct *vma, unsigned long addr,
261 void *buf, int len, int write);
Andy Lutomirski78d683e2014-05-19 15:58:32 -0700262
263 /* Called by the /proc/PID/maps code to ask the vma whether it
264 * has a special name. Returning non-NULL will also cause this
265 * vma to be dumped unconditionally. */
266 const char *(*name)(struct vm_area_struct *vma);
267
Linus Torvalds1da177e2005-04-16 15:20:36 -0700268#ifdef CONFIG_NUMA
Lee Schermerhorna6020ed2008-04-28 02:13:14 -0700269 /*
270 * set_policy() op must add a reference to any non-NULL @new mempolicy
271 * to hold the policy upon return. Caller should pass NULL @new to
272 * remove a policy and fall back to surrounding context--i.e. do not
273 * install a MPOL_DEFAULT policy, nor the task or system default
274 * mempolicy.
275 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276 int (*set_policy)(struct vm_area_struct *vma, struct mempolicy *new);
Lee Schermerhorna6020ed2008-04-28 02:13:14 -0700277
278 /*
279 * get_policy() op must add reference [mpol_get()] to any policy at
280 * (vma,addr) marked as MPOL_SHARED. The shared policy infrastructure
281 * in mm/mempolicy.c will do this automatically.
282 * get_policy() must NOT add a ref if the policy at (vma,addr) is not
283 * marked as MPOL_SHARED. vma policies are protected by the mmap_sem.
284 * If no [shared/vma] mempolicy exists at the addr, get_policy() op
285 * must return NULL--i.e., do not "fallback" to task or system default
286 * policy.
287 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700288 struct mempolicy *(*get_policy)(struct vm_area_struct *vma,
289 unsigned long addr);
290#endif
David Vrabel667a0a02014-12-18 14:48:15 +0000291 /*
292 * Called by vm_normal_page() for special PTEs to find the
293 * page for @addr. This is useful if the default behavior
294 * (using pte_page()) would not find the correct page.
295 */
296 struct page *(*find_special_page)(struct vm_area_struct *vma,
297 unsigned long addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298};
299
300struct mmu_gather;
301struct inode;
302
Andrew Morton349aef02006-01-08 01:04:36 -0800303#define page_private(page) ((page)->private)
304#define set_page_private(page, v) ((page)->private = (v))
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700305
Minchan Kimb12c4ad2012-10-08 16:32:08 -0700306/* It's valid only if the page is free path or free_list */
307static inline void set_freepage_migratetype(struct page *page, int migratetype)
308{
Minchan Kim95e34412012-10-08 16:32:11 -0700309 page->index = migratetype;
Minchan Kimb12c4ad2012-10-08 16:32:08 -0700310}
311
312/* It's valid only if the page is free path or free_list */
313static inline int get_freepage_migratetype(struct page *page)
314{
Minchan Kim95e34412012-10-08 16:32:11 -0700315 return page->index;
Minchan Kimb12c4ad2012-10-08 16:32:08 -0700316}
317
Linus Torvalds1da177e2005-04-16 15:20:36 -0700318/*
319 * FIXME: take this include out, include page-flags.h in
320 * files which need it (119 of them)
321 */
322#include <linux/page-flags.h>
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800323#include <linux/huge_mm.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324
325/*
326 * Methods to modify the page usage count.
327 *
328 * What counts for a page usage:
329 * - cache mapping (page->mapping)
330 * - private data (page->private)
331 * - page mapped in a task's page tables, each mapping
332 * is counted separately
333 *
334 * Also, many kernel routines increase the page count before a critical
335 * routine so they can be sure the page doesn't go away from under them.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336 */
337
338/*
Nick Pigginda6052f2006-09-25 23:31:35 -0700339 * Drop a ref, return true if the refcount fell to zero (the page has no users)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700340 */
Nick Piggin7c8ee9a2006-03-22 00:08:03 -0800341static inline int put_page_testzero(struct page *page)
342{
Sasha Levin309381fea2014-01-23 15:52:54 -0800343 VM_BUG_ON_PAGE(atomic_read(&page->_count) == 0, page);
Nick Piggin8dc04ef2006-03-22 00:08:03 -0800344 return atomic_dec_and_test(&page->_count);
Nick Piggin7c8ee9a2006-03-22 00:08:03 -0800345}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700346
347/*
Nick Piggin7c8ee9a2006-03-22 00:08:03 -0800348 * Try to grab a ref unless the page has a refcount of zero, return false if
349 * that is the case.
Alexey Kardashevskiy8e0861f2013-08-28 18:37:42 +1000350 * This can be called when MMU is off so it must not access
351 * any of the virtual mappings.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700352 */
Nick Piggin7c8ee9a2006-03-22 00:08:03 -0800353static inline int get_page_unless_zero(struct page *page)
354{
Nick Piggin8dc04ef2006-03-22 00:08:03 -0800355 return atomic_inc_not_zero(&page->_count);
Nick Piggin7c8ee9a2006-03-22 00:08:03 -0800356}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700357
Alexey Kardashevskiy8e0861f2013-08-28 18:37:42 +1000358/*
359 * Try to drop a ref unless the page has a refcount of one, return false if
360 * that is the case.
361 * This is to make sure that the refcount won't become zero after this drop.
362 * This can be called when MMU is off so it must not access
363 * any of the virtual mappings.
364 */
365static inline int put_page_unless_one(struct page *page)
366{
367 return atomic_add_unless(&page->_count, -1, 1);
368}
369
Wu Fengguang53df8fd2010-01-27 11:06:39 +0800370extern int page_is_ram(unsigned long pfn);
Mike Travis67cf13c2014-10-13 15:54:03 -0700371extern int region_is_ram(resource_size_t phys_addr, unsigned long size);
Wu Fengguang53df8fd2010-01-27 11:06:39 +0800372
Christoph Lameter48667e72008-02-04 22:28:31 -0800373/* Support for virtually mapped pages */
Christoph Lameterb3bdda02008-02-04 22:28:32 -0800374struct page *vmalloc_to_page(const void *addr);
375unsigned long vmalloc_to_pfn(const void *addr);
Christoph Lameter48667e72008-02-04 22:28:31 -0800376
Paul Mundt0738c4b2008-03-12 16:51:31 +0900377/*
378 * Determine if an address is within the vmalloc range
379 *
380 * On nommu, vmalloc/vfree wrap through kmalloc/kfree directly, so there
381 * is no special casing required.
382 */
Christoph Lameter9e2779f2008-02-04 22:28:34 -0800383static inline int is_vmalloc_addr(const void *x)
384{
Paul Mundt0738c4b2008-03-12 16:51:31 +0900385#ifdef CONFIG_MMU
Christoph Lameter9e2779f2008-02-04 22:28:34 -0800386 unsigned long addr = (unsigned long)x;
387
388 return addr >= VMALLOC_START && addr < VMALLOC_END;
Paul Mundt0738c4b2008-03-12 16:51:31 +0900389#else
390 return 0;
David Howells8ca3ed82008-02-23 15:23:37 -0800391#endif
Paul Mundt0738c4b2008-03-12 16:51:31 +0900392}
KAMEZAWA Hiroyuki81ac3ad2009-09-22 16:45:49 -0700393#ifdef CONFIG_MMU
394extern int is_vmalloc_or_module_addr(const void *x);
395#else
David Howells934831d2009-09-24 12:33:32 +0100396static inline int is_vmalloc_or_module_addr(const void *x)
KAMEZAWA Hiroyuki81ac3ad2009-09-22 16:45:49 -0700397{
398 return 0;
399}
400#endif
Christoph Lameter9e2779f2008-02-04 22:28:34 -0800401
Al Viro39f1f782014-05-06 14:02:53 -0400402extern void kvfree(const void *addr);
403
Andrea Arcangelie9da73d2011-01-13 15:46:32 -0800404static inline void compound_lock(struct page *page)
405{
406#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Sasha Levin309381fea2014-01-23 15:52:54 -0800407 VM_BUG_ON_PAGE(PageSlab(page), page);
Andrea Arcangelie9da73d2011-01-13 15:46:32 -0800408 bit_spin_lock(PG_compound_lock, &page->flags);
409#endif
410}
411
412static inline void compound_unlock(struct page *page)
413{
414#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Sasha Levin309381fea2014-01-23 15:52:54 -0800415 VM_BUG_ON_PAGE(PageSlab(page), page);
Andrea Arcangelie9da73d2011-01-13 15:46:32 -0800416 bit_spin_unlock(PG_compound_lock, &page->flags);
417#endif
418}
419
420static inline unsigned long compound_lock_irqsave(struct page *page)
421{
422 unsigned long uninitialized_var(flags);
423#ifdef CONFIG_TRANSPARENT_HUGEPAGE
424 local_irq_save(flags);
425 compound_lock(page);
426#endif
427 return flags;
428}
429
430static inline void compound_unlock_irqrestore(struct page *page,
431 unsigned long flags)
432{
433#ifdef CONFIG_TRANSPARENT_HUGEPAGE
434 compound_unlock(page);
435 local_irq_restore(flags);
436#endif
437}
438
Jianyu Zhand2ee40e2014-06-04 16:08:02 -0700439static inline struct page *compound_head_by_tail(struct page *tail)
440{
441 struct page *head = tail->first_page;
442
443 /*
444 * page->first_page may be a dangling pointer to an old
445 * compound page, so recheck that it is still a tail
446 * page before returning.
447 */
448 smp_rmb();
449 if (likely(PageTail(tail)))
450 return head;
451 return tail;
452}
453
Joonsoo Kimccaafd72015-02-10 14:09:35 -0800454/*
455 * Since either compound page could be dismantled asynchronously in THP
456 * or we access asynchronously arbitrary positioned struct page, there
457 * would be tail flag race. To handle this race, we should call
458 * smp_rmb() before checking tail flag. compound_head_by_tail() did it.
459 */
Christoph Lameterd85f3382007-05-06 14:49:39 -0700460static inline struct page *compound_head(struct page *page)
461{
Jianyu Zhand2ee40e2014-06-04 16:08:02 -0700462 if (unlikely(PageTail(page)))
463 return compound_head_by_tail(page);
Christoph Lameterd85f3382007-05-06 14:49:39 -0700464 return page;
465}
466
Andrea Arcangeli70b50f92011-11-02 13:36:59 -0700467/*
Joonsoo Kimccaafd72015-02-10 14:09:35 -0800468 * If we access compound page synchronously such as access to
469 * allocated page, there is no need to handle tail flag race, so we can
470 * check tail flag directly without any synchronization primitive.
471 */
472static inline struct page *compound_head_fast(struct page *page)
473{
474 if (unlikely(PageTail(page)))
475 return page->first_page;
476 return page;
477}
478
479/*
Andrea Arcangeli70b50f92011-11-02 13:36:59 -0700480 * The atomic page->_mapcount, starts from -1: so that transitions
481 * both from it and to it can be tracked, using atomic_inc_and_test
482 * and atomic_add_negative(-1).
483 */
Mel Gorman22b751c2013-02-22 16:34:59 -0800484static inline void page_mapcount_reset(struct page *page)
Andrea Arcangeli70b50f92011-11-02 13:36:59 -0700485{
486 atomic_set(&(page)->_mapcount, -1);
487}
488
489static inline int page_mapcount(struct page *page)
490{
Wang, Yalin1d148e22015-02-11 15:24:48 -0800491 VM_BUG_ON_PAGE(PageSlab(page), page);
492 return atomic_read(&page->_mapcount) + 1;
Andrea Arcangeli70b50f92011-11-02 13:36:59 -0700493}
494
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700495static inline int page_count(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496{
Christoph Lameterd85f3382007-05-06 14:49:39 -0700497 return atomic_read(&compound_head(page)->_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700498}
499
Andrea Arcangeli44518d22014-01-21 15:48:54 -0800500static inline bool __compound_tail_refcounted(struct page *page)
501{
Linus Torvaldsf3ca10d2015-04-22 09:44:36 -0700502 return !PageSlab(page) && !PageHeadHuge(page);
Andrea Arcangeli44518d22014-01-21 15:48:54 -0800503}
504
505/*
506 * This takes a head page as parameter and tells if the
507 * tail page reference counting can be skipped.
508 *
509 * For this to be safe, PageSlab and PageHeadHuge must remain true on
510 * any given page where they return true here, until all tail pins
511 * have been released.
512 */
513static inline bool compound_tail_refcounted(struct page *page)
514{
Sasha Levin309381fea2014-01-23 15:52:54 -0800515 VM_BUG_ON_PAGE(!PageHead(page), page);
Andrea Arcangeli44518d22014-01-21 15:48:54 -0800516 return __compound_tail_refcounted(page);
517}
518
Andrea Arcangelib35a35b2011-11-02 13:37:36 -0700519static inline void get_huge_page_tail(struct page *page)
520{
521 /*
Oleg Nesterov5eaf1a92014-01-21 15:49:04 -0800522 * __split_huge_page_refcount() cannot run from under us.
Andrea Arcangelib35a35b2011-11-02 13:37:36 -0700523 */
Sasha Levin309381fea2014-01-23 15:52:54 -0800524 VM_BUG_ON_PAGE(!PageTail(page), page);
525 VM_BUG_ON_PAGE(page_mapcount(page) < 0, page);
526 VM_BUG_ON_PAGE(atomic_read(&page->_count) != 0, page);
Oleg Nesterov5eaf1a92014-01-21 15:49:04 -0800527 if (compound_tail_refcounted(page->first_page))
Andrea Arcangeli44518d22014-01-21 15:48:54 -0800528 atomic_inc(&page->_mapcount);
Andrea Arcangelib35a35b2011-11-02 13:37:36 -0700529}
530
Andrea Arcangeli70b50f92011-11-02 13:36:59 -0700531extern bool __get_page_tail(struct page *page);
532
Linus Torvalds1da177e2005-04-16 15:20:36 -0700533static inline void get_page(struct page *page)
534{
Andrea Arcangeli70b50f92011-11-02 13:36:59 -0700535 if (unlikely(PageTail(page)))
536 if (likely(__get_page_tail(page)))
537 return;
Andrea Arcangeli91807062011-01-13 15:46:32 -0800538 /*
539 * Getting a normal page or the head of a compound page
Andrea Arcangeli70b50f92011-11-02 13:36:59 -0700540 * requires to already have an elevated page->_count.
Andrea Arcangeli91807062011-01-13 15:46:32 -0800541 */
Sasha Levin309381fea2014-01-23 15:52:54 -0800542 VM_BUG_ON_PAGE(atomic_read(&page->_count) <= 0, page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700543 atomic_inc(&page->_count);
544}
545
Christoph Lameterb49af682007-05-06 14:49:41 -0700546static inline struct page *virt_to_head_page(const void *x)
547{
548 struct page *page = virt_to_page(x);
Joonsoo Kimccaafd72015-02-10 14:09:35 -0800549
550 /*
551 * We don't need to worry about synchronization of tail flag
552 * when we call virt_to_head_page() since it is only called for
553 * already allocated page and this page won't be freed until
554 * this virt_to_head_page() is finished. So use _fast variant.
555 */
556 return compound_head_fast(page);
Christoph Lameterb49af682007-05-06 14:49:41 -0700557}
558
Nick Piggin7835e982006-03-22 00:08:40 -0800559/*
560 * Setup the page count before being freed into the page allocator for
561 * the first time (boot or memory hotplug)
562 */
563static inline void init_page_count(struct page *page)
564{
565 atomic_set(&page->_count, 1);
566}
567
Linus Torvalds1da177e2005-04-16 15:20:36 -0700568void put_page(struct page *page);
Alexander Zarochentsev1d7ea732006-08-13 23:24:27 -0700569void put_pages_list(struct list_head *pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700570
Nick Piggin8dfcc9b2006-03-22 00:08:05 -0800571void split_page(struct page *page, unsigned int order);
Mel Gorman748446b2010-05-24 14:32:27 -0700572int split_free_page(struct page *page);
Nick Piggin8dfcc9b2006-03-22 00:08:05 -0800573
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574/*
Andy Whitcroft33f2ef82006-12-06 20:33:32 -0800575 * Compound pages have a destructor function. Provide a
576 * prototype for that function and accessor functions.
577 * These are _only_ valid on the head of a PG_compound page.
578 */
Andy Whitcroft33f2ef82006-12-06 20:33:32 -0800579
580static inline void set_compound_page_dtor(struct page *page,
581 compound_page_dtor *dtor)
582{
Kirill A. Shutemove4b294c2015-02-11 15:24:46 -0800583 page[1].compound_dtor = dtor;
Andy Whitcroft33f2ef82006-12-06 20:33:32 -0800584}
585
586static inline compound_page_dtor *get_compound_page_dtor(struct page *page)
587{
Kirill A. Shutemove4b294c2015-02-11 15:24:46 -0800588 return page[1].compound_dtor;
Andy Whitcroft33f2ef82006-12-06 20:33:32 -0800589}
590
Christoph Lameterd85f3382007-05-06 14:49:39 -0700591static inline int compound_order(struct page *page)
592{
Christoph Lameter6d777952007-05-06 14:49:40 -0700593 if (!PageHead(page))
Christoph Lameterd85f3382007-05-06 14:49:39 -0700594 return 0;
Kirill A. Shutemove4b294c2015-02-11 15:24:46 -0800595 return page[1].compound_order;
Christoph Lameterd85f3382007-05-06 14:49:39 -0700596}
597
598static inline void set_compound_order(struct page *page, unsigned long order)
599{
Kirill A. Shutemove4b294c2015-02-11 15:24:46 -0800600 page[1].compound_order = order;
Christoph Lameterd85f3382007-05-06 14:49:39 -0700601}
602
Michal Simek3dece372011-01-21 08:49:56 +0100603#ifdef CONFIG_MMU
Andy Whitcroft33f2ef82006-12-06 20:33:32 -0800604/*
Andrea Arcangeli14fd4032011-01-13 15:46:37 -0800605 * Do pte_mkwrite, but only if the vma says VM_WRITE. We do this when
606 * servicing faults for write access. In the normal case, do always want
607 * pte_mkwrite. But get_user_pages can cause write faults for mappings
608 * that do not have writing enabled, when used by access_process_vm.
609 */
610static inline pte_t maybe_mkwrite(pte_t pte, struct vm_area_struct *vma)
611{
612 if (likely(vma->vm_flags & VM_WRITE))
613 pte = pte_mkwrite(pte);
614 return pte;
615}
Kirill A. Shutemov8c6e50b2014-04-07 15:37:18 -0700616
617void do_set_pte(struct vm_area_struct *vma, unsigned long address,
618 struct page *page, pte_t *pte, bool write, bool anon);
Michal Simek3dece372011-01-21 08:49:56 +0100619#endif
Andrea Arcangeli14fd4032011-01-13 15:46:37 -0800620
621/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700622 * Multiple processes may "see" the same page. E.g. for untouched
623 * mappings of /dev/null, all processes see the same page full of
624 * zeroes, and text pages of executables and shared libraries have
625 * only one copy in memory, at most, normally.
626 *
627 * For the non-reserved pages, page_count(page) denotes a reference count.
Paolo 'Blaisorblade' Giarrusso7e871b62005-09-21 09:55:38 -0700628 * page_count() == 0 means the page is free. page->lru is then used for
629 * freelist management in the buddy allocator.
Nick Pigginda6052f2006-09-25 23:31:35 -0700630 * page_count() > 0 means the page has been allocated.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700631 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700632 * Pages are allocated by the slab allocator in order to provide memory
633 * to kmalloc and kmem_cache_alloc. In this case, the management of the
634 * page, and the fields in 'struct page' are the responsibility of mm/slab.c
635 * unless a particular usage is carefully commented. (the responsibility of
636 * freeing the kmalloc memory is the caller's, of course).
Linus Torvalds1da177e2005-04-16 15:20:36 -0700637 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700638 * A page may be used by anyone else who does a __get_free_page().
639 * In this case, page_count still tracks the references, and should only
640 * be used through the normal accessor functions. The top bits of page->flags
641 * and page->virtual store page management information, but all other fields
642 * are unused and could be used privately, carefully. The management of this
643 * page is the responsibility of the one who allocated it, and those who have
644 * subsequently been given references to it.
645 *
646 * The other pages (we may call them "pagecache pages") are completely
Linus Torvalds1da177e2005-04-16 15:20:36 -0700647 * managed by the Linux memory manager: I/O, buffers, swapping etc.
648 * The following discussion applies only to them.
649 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700650 * A pagecache page contains an opaque `private' member, which belongs to the
651 * page's address_space. Usually, this is the address of a circular list of
652 * the page's disk buffers. PG_private must be set to tell the VM to call
653 * into the filesystem to release these pages.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700654 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700655 * A page may belong to an inode's memory mapping. In this case, page->mapping
656 * is the pointer to the inode, and page->index is the file offset of the page,
657 * in units of PAGE_CACHE_SIZE.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700659 * If pagecache pages are not associated with an inode, they are said to be
660 * anonymous pages. These may become associated with the swapcache, and in that
661 * case PG_swapcache is set, and page->private is an offset into the swapcache.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700662 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700663 * In either case (swapcache or inode backed), the pagecache itself holds one
664 * reference to the page. Setting PG_private should also increment the
665 * refcount. The each user mapping also has a reference to the page.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700666 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700667 * The pagecache pages are stored in a per-mapping radix tree, which is
668 * rooted at mapping->page_tree, and indexed by offset.
669 * Where 2.4 and early 2.6 kernels kept dirty/clean pages in per-address_space
670 * lists, we instead now tag pages as dirty/writeback in the radix tree.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671 *
Nick Pigginda6052f2006-09-25 23:31:35 -0700672 * All pagecache pages may be subject to I/O:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700673 * - inode pages may need to be read from disk,
674 * - inode pages which have been modified and are MAP_SHARED may need
Nick Pigginda6052f2006-09-25 23:31:35 -0700675 * to be written back to the inode on disk,
676 * - anonymous pages (including MAP_PRIVATE file mappings) which have been
677 * modified may need to be swapped out to swap space and (later) to be read
678 * back into memory.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700679 */
680
681/*
682 * The zone field is never updated after free_area_init_core()
683 * sets it, so none of the operations on it need to be atomic.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700684 */
Dave Hansen348f8b62005-06-23 00:07:40 -0700685
Peter Zijlstra90572892013-10-07 11:29:20 +0100686/* Page flags: | [SECTION] | [NODE] | ZONE | [LAST_CPUPID] | ... | FLAGS | */
Andi Kleen07808b72005-11-05 17:25:53 +0100687#define SECTIONS_PGOFF ((sizeof(unsigned long)*8) - SECTIONS_WIDTH)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700688#define NODES_PGOFF (SECTIONS_PGOFF - NODES_WIDTH)
689#define ZONES_PGOFF (NODES_PGOFF - ZONES_WIDTH)
Peter Zijlstra90572892013-10-07 11:29:20 +0100690#define LAST_CPUPID_PGOFF (ZONES_PGOFF - LAST_CPUPID_WIDTH)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700691
692/*
Lucas De Marchi25985ed2011-03-30 22:57:33 -0300693 * Define the bit shifts to access each section. For non-existent
Dave Hansen348f8b62005-06-23 00:07:40 -0700694 * sections we define the shift as 0; that plus a 0 mask ensures
695 * the compiler will optimise away reference to them.
696 */
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700697#define SECTIONS_PGSHIFT (SECTIONS_PGOFF * (SECTIONS_WIDTH != 0))
698#define NODES_PGSHIFT (NODES_PGOFF * (NODES_WIDTH != 0))
699#define ZONES_PGSHIFT (ZONES_PGOFF * (ZONES_WIDTH != 0))
Peter Zijlstra90572892013-10-07 11:29:20 +0100700#define LAST_CPUPID_PGSHIFT (LAST_CPUPID_PGOFF * (LAST_CPUPID_WIDTH != 0))
Dave Hansen348f8b62005-06-23 00:07:40 -0700701
Will Deaconbce54bb2010-10-26 14:21:37 -0700702/* NODE:ZONE or SECTION:ZONE is used to ID a zone for the buddy allocator */
703#ifdef NODE_NOT_IN_PAGE_FLAGS
Christoph Lameter89689ae2006-12-06 20:31:45 -0800704#define ZONEID_SHIFT (SECTIONS_SHIFT + ZONES_SHIFT)
Andy Whitcroftbd8029b2007-02-10 01:43:14 -0800705#define ZONEID_PGOFF ((SECTIONS_PGOFF < ZONES_PGOFF)? \
706 SECTIONS_PGOFF : ZONES_PGOFF)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700707#else
Christoph Lameter89689ae2006-12-06 20:31:45 -0800708#define ZONEID_SHIFT (NODES_SHIFT + ZONES_SHIFT)
Andy Whitcroftbd8029b2007-02-10 01:43:14 -0800709#define ZONEID_PGOFF ((NODES_PGOFF < ZONES_PGOFF)? \
710 NODES_PGOFF : ZONES_PGOFF)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700711#endif
Christoph Lameter89689ae2006-12-06 20:31:45 -0800712
Andy Whitcroftbd8029b2007-02-10 01:43:14 -0800713#define ZONEID_PGSHIFT (ZONEID_PGOFF * (ZONEID_SHIFT != 0))
Dave Hansen348f8b62005-06-23 00:07:40 -0700714
Christoph Lameter9223b412008-04-28 02:12:48 -0700715#if SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
716#error SECTIONS_WIDTH+NODES_WIDTH+ZONES_WIDTH > BITS_PER_LONG - NR_PAGEFLAGS
Dave Hansen348f8b62005-06-23 00:07:40 -0700717#endif
718
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700719#define ZONES_MASK ((1UL << ZONES_WIDTH) - 1)
720#define NODES_MASK ((1UL << NODES_WIDTH) - 1)
721#define SECTIONS_MASK ((1UL << SECTIONS_WIDTH) - 1)
Srikar Dronamraju834a9642014-04-07 15:37:57 -0700722#define LAST_CPUPID_MASK ((1UL << LAST_CPUPID_SHIFT) - 1)
Christoph Lameter89689ae2006-12-06 20:31:45 -0800723#define ZONEID_MASK ((1UL << ZONEID_SHIFT) - 1)
Dave Hansen348f8b62005-06-23 00:07:40 -0700724
Ian Campbell33dd4e02011-07-25 17:11:51 -0700725static inline enum zone_type page_zonenum(const struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700726{
Dave Hansen348f8b62005-06-23 00:07:40 -0700727 return (page->flags >> ZONES_PGSHIFT) & ZONES_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729
Cody P Schafer9127ab42013-02-22 16:35:21 -0800730#if defined(CONFIG_SPARSEMEM) && !defined(CONFIG_SPARSEMEM_VMEMMAP)
731#define SECTION_IN_PAGE_FLAGS
732#endif
733
Christoph Lameter89689ae2006-12-06 20:31:45 -0800734/*
Vlastimil Babka7a8010c2013-09-11 14:22:35 -0700735 * The identification function is mainly used by the buddy allocator for
736 * determining if two pages could be buddies. We are not really identifying
737 * the zone since we could be using the section number id if we do not have
738 * node id available in page flags.
739 * We only guarantee that it will return the same value for two combinable
740 * pages in a zone.
Christoph Lameter89689ae2006-12-06 20:31:45 -0800741 */
Andy Whitcroftcb2b95e2006-06-23 02:03:01 -0700742static inline int page_zone_id(struct page *page)
743{
Christoph Lameter89689ae2006-12-06 20:31:45 -0800744 return (page->flags >> ZONEID_PGSHIFT) & ZONEID_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700745}
746
Andy Whitcroft25ba77c2006-12-06 20:33:03 -0800747static inline int zone_to_nid(struct zone *zone)
Christoph Lameter89fa3022006-09-25 23:31:55 -0700748{
Christoph Lameterd5f541e2006-09-27 01:50:08 -0700749#ifdef CONFIG_NUMA
750 return zone->node;
751#else
752 return 0;
753#endif
Christoph Lameter89fa3022006-09-25 23:31:55 -0700754}
755
Christoph Lameter89689ae2006-12-06 20:31:45 -0800756#ifdef NODE_NOT_IN_PAGE_FLAGS
Ian Campbell33dd4e02011-07-25 17:11:51 -0700757extern int page_to_nid(const struct page *page);
Christoph Lameter89689ae2006-12-06 20:31:45 -0800758#else
Ian Campbell33dd4e02011-07-25 17:11:51 -0700759static inline int page_to_nid(const struct page *page)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700760{
Christoph Lameter89689ae2006-12-06 20:31:45 -0800761 return (page->flags >> NODES_PGSHIFT) & NODES_MASK;
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700762}
Christoph Lameter89689ae2006-12-06 20:31:45 -0800763#endif
764
Mel Gorman57e0a032012-11-12 09:06:20 +0000765#ifdef CONFIG_NUMA_BALANCING
Peter Zijlstra90572892013-10-07 11:29:20 +0100766static inline int cpu_pid_to_cpupid(int cpu, int pid)
Mel Gorman57e0a032012-11-12 09:06:20 +0000767{
Peter Zijlstra90572892013-10-07 11:29:20 +0100768 return ((cpu & LAST__CPU_MASK) << LAST__PID_SHIFT) | (pid & LAST__PID_MASK);
Mel Gorman57e0a032012-11-12 09:06:20 +0000769}
770
Peter Zijlstra90572892013-10-07 11:29:20 +0100771static inline int cpupid_to_pid(int cpupid)
Mel Gorman57e0a032012-11-12 09:06:20 +0000772{
Peter Zijlstra90572892013-10-07 11:29:20 +0100773 return cpupid & LAST__PID_MASK;
Mel Gorman57e0a032012-11-12 09:06:20 +0000774}
Mel Gormanb7958542013-10-07 11:29:07 +0100775
Peter Zijlstra90572892013-10-07 11:29:20 +0100776static inline int cpupid_to_cpu(int cpupid)
Mel Gorman57e0a032012-11-12 09:06:20 +0000777{
Peter Zijlstra90572892013-10-07 11:29:20 +0100778 return (cpupid >> LAST__PID_SHIFT) & LAST__CPU_MASK;
Mel Gormanb7958542013-10-07 11:29:07 +0100779}
780
Peter Zijlstra90572892013-10-07 11:29:20 +0100781static inline int cpupid_to_nid(int cpupid)
Mel Gormanb7958542013-10-07 11:29:07 +0100782{
Peter Zijlstra90572892013-10-07 11:29:20 +0100783 return cpu_to_node(cpupid_to_cpu(cpupid));
Mel Gormanb7958542013-10-07 11:29:07 +0100784}
785
Peter Zijlstra90572892013-10-07 11:29:20 +0100786static inline bool cpupid_pid_unset(int cpupid)
Mel Gormanb7958542013-10-07 11:29:07 +0100787{
Peter Zijlstra90572892013-10-07 11:29:20 +0100788 return cpupid_to_pid(cpupid) == (-1 & LAST__PID_MASK);
Mel Gormanb7958542013-10-07 11:29:07 +0100789}
790
Peter Zijlstra90572892013-10-07 11:29:20 +0100791static inline bool cpupid_cpu_unset(int cpupid)
Mel Gormanb7958542013-10-07 11:29:07 +0100792{
Peter Zijlstra90572892013-10-07 11:29:20 +0100793 return cpupid_to_cpu(cpupid) == (-1 & LAST__CPU_MASK);
Mel Gormanb7958542013-10-07 11:29:07 +0100794}
795
Peter Zijlstra8c8a7432013-10-07 11:29:21 +0100796static inline bool __cpupid_match_pid(pid_t task_pid, int cpupid)
797{
798 return (task_pid & LAST__PID_MASK) == cpupid_to_pid(cpupid);
799}
800
801#define cpupid_match_pid(task, cpupid) __cpupid_match_pid(task->pid, cpupid)
Peter Zijlstra90572892013-10-07 11:29:20 +0100802#ifdef LAST_CPUPID_NOT_IN_PAGE_FLAGS
803static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
Mel Gormanb7958542013-10-07 11:29:07 +0100804{
Liu Ping Fan1ae71d02014-03-03 15:38:39 -0800805 return xchg(&page->_last_cpupid, cpupid & LAST_CPUPID_MASK);
Mel Gormanb7958542013-10-07 11:29:07 +0100806}
Peter Zijlstra90572892013-10-07 11:29:20 +0100807
808static inline int page_cpupid_last(struct page *page)
Mel Gormanb7958542013-10-07 11:29:07 +0100809{
Peter Zijlstra90572892013-10-07 11:29:20 +0100810 return page->_last_cpupid;
811}
812static inline void page_cpupid_reset_last(struct page *page)
813{
Liu Ping Fan1ae71d02014-03-03 15:38:39 -0800814 page->_last_cpupid = -1 & LAST_CPUPID_MASK;
Mel Gorman57e0a032012-11-12 09:06:20 +0000815}
816#else
Peter Zijlstra90572892013-10-07 11:29:20 +0100817static inline int page_cpupid_last(struct page *page)
Peter Zijlstra75980e92013-02-22 16:34:32 -0800818{
Peter Zijlstra90572892013-10-07 11:29:20 +0100819 return (page->flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
Peter Zijlstra75980e92013-02-22 16:34:32 -0800820}
821
Peter Zijlstra90572892013-10-07 11:29:20 +0100822extern int page_cpupid_xchg_last(struct page *page, int cpupid);
Peter Zijlstra75980e92013-02-22 16:34:32 -0800823
Peter Zijlstra90572892013-10-07 11:29:20 +0100824static inline void page_cpupid_reset_last(struct page *page)
Peter Zijlstra75980e92013-02-22 16:34:32 -0800825{
Peter Zijlstra90572892013-10-07 11:29:20 +0100826 int cpupid = (1 << LAST_CPUPID_SHIFT) - 1;
Mel Gorman4468b8f2013-02-22 16:34:46 -0800827
Peter Zijlstra90572892013-10-07 11:29:20 +0100828 page->flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);
829 page->flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;
Peter Zijlstra75980e92013-02-22 16:34:32 -0800830}
Peter Zijlstra90572892013-10-07 11:29:20 +0100831#endif /* LAST_CPUPID_NOT_IN_PAGE_FLAGS */
832#else /* !CONFIG_NUMA_BALANCING */
833static inline int page_cpupid_xchg_last(struct page *page, int cpupid)
Mel Gorman57e0a032012-11-12 09:06:20 +0000834{
Peter Zijlstra90572892013-10-07 11:29:20 +0100835 return page_to_nid(page); /* XXX */
Mel Gorman57e0a032012-11-12 09:06:20 +0000836}
837
Peter Zijlstra90572892013-10-07 11:29:20 +0100838static inline int page_cpupid_last(struct page *page)
Mel Gorman57e0a032012-11-12 09:06:20 +0000839{
Peter Zijlstra90572892013-10-07 11:29:20 +0100840 return page_to_nid(page); /* XXX */
Mel Gorman57e0a032012-11-12 09:06:20 +0000841}
842
Peter Zijlstra90572892013-10-07 11:29:20 +0100843static inline int cpupid_to_nid(int cpupid)
Mel Gormanb7958542013-10-07 11:29:07 +0100844{
845 return -1;
846}
847
Peter Zijlstra90572892013-10-07 11:29:20 +0100848static inline int cpupid_to_pid(int cpupid)
Mel Gormanb7958542013-10-07 11:29:07 +0100849{
850 return -1;
851}
852
Peter Zijlstra90572892013-10-07 11:29:20 +0100853static inline int cpupid_to_cpu(int cpupid)
Mel Gormanb7958542013-10-07 11:29:07 +0100854{
855 return -1;
856}
857
Peter Zijlstra90572892013-10-07 11:29:20 +0100858static inline int cpu_pid_to_cpupid(int nid, int pid)
859{
860 return -1;
861}
862
863static inline bool cpupid_pid_unset(int cpupid)
Mel Gormanb7958542013-10-07 11:29:07 +0100864{
865 return 1;
866}
867
Peter Zijlstra90572892013-10-07 11:29:20 +0100868static inline void page_cpupid_reset_last(struct page *page)
Mel Gorman57e0a032012-11-12 09:06:20 +0000869{
870}
Peter Zijlstra8c8a7432013-10-07 11:29:21 +0100871
872static inline bool cpupid_match_pid(struct task_struct *task, int cpupid)
873{
874 return false;
875}
Peter Zijlstra90572892013-10-07 11:29:20 +0100876#endif /* CONFIG_NUMA_BALANCING */
Mel Gorman57e0a032012-11-12 09:06:20 +0000877
Ian Campbell33dd4e02011-07-25 17:11:51 -0700878static inline struct zone *page_zone(const struct page *page)
Christoph Lameter89689ae2006-12-06 20:31:45 -0800879{
880 return &NODE_DATA(page_to_nid(page))->node_zones[page_zonenum(page)];
881}
882
Cody P Schafer9127ab42013-02-22 16:35:21 -0800883#ifdef SECTION_IN_PAGE_FLAGS
Daniel Kiperbf4e8902011-05-24 17:12:32 -0700884static inline void set_page_section(struct page *page, unsigned long section)
885{
886 page->flags &= ~(SECTIONS_MASK << SECTIONS_PGSHIFT);
887 page->flags |= (section & SECTIONS_MASK) << SECTIONS_PGSHIFT;
888}
889
Ian Campbellaa462ab2011-08-17 17:40:33 +0100890static inline unsigned long page_to_section(const struct page *page)
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700891{
892 return (page->flags >> SECTIONS_PGSHIFT) & SECTIONS_MASK;
893}
Christoph Lameter308c05e2008-04-28 02:12:43 -0700894#endif
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700895
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700896static inline void set_page_zone(struct page *page, enum zone_type zone)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700897{
Dave Hansen348f8b62005-06-23 00:07:40 -0700898 page->flags &= ~(ZONES_MASK << ZONES_PGSHIFT);
899 page->flags |= (zone & ZONES_MASK) << ZONES_PGSHIFT;
900}
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700901
Dave Hansen348f8b62005-06-23 00:07:40 -0700902static inline void set_page_node(struct page *page, unsigned long node)
903{
904 page->flags &= ~(NODES_MASK << NODES_PGSHIFT);
905 page->flags |= (node & NODES_MASK) << NODES_PGSHIFT;
906}
Christoph Lameter89689ae2006-12-06 20:31:45 -0800907
Christoph Lameter2f1b6242006-09-25 23:31:13 -0700908static inline void set_page_links(struct page *page, enum zone_type zone,
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700909 unsigned long node, unsigned long pfn)
Dave Hansen348f8b62005-06-23 00:07:40 -0700910{
911 set_page_zone(page, zone);
912 set_page_node(page, node);
Cody P Schafer9127ab42013-02-22 16:35:21 -0800913#ifdef SECTION_IN_PAGE_FLAGS
Andy Whitcroftd41dee32005-06-23 00:07:54 -0700914 set_page_section(page, pfn_to_section_nr(pfn));
Daniel Kiperbf4e8902011-05-24 17:12:32 -0700915#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916}
917
Christoph Lameterf6ac2352006-06-30 01:55:32 -0700918/*
919 * Some inline functions in vmstat.h depend on page_zone()
920 */
921#include <linux/vmstat.h>
922
Ian Campbell33dd4e02011-07-25 17:11:51 -0700923static __always_inline void *lowmem_page_address(const struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924{
Ian Campbellaa462ab2011-08-17 17:40:33 +0100925 return __va(PFN_PHYS(page_to_pfn(page)));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700926}
927
928#if defined(CONFIG_HIGHMEM) && !defined(WANT_PAGE_VIRTUAL)
929#define HASHED_PAGE_VIRTUAL
930#endif
931
932#if defined(WANT_PAGE_VIRTUAL)
Geert Uytterhoevenf92f4552014-01-21 15:48:47 -0800933static inline void *page_address(const struct page *page)
934{
935 return page->virtual;
936}
937static inline void set_page_address(struct page *page, void *address)
938{
939 page->virtual = address;
940}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700941#define page_address_init() do { } while(0)
942#endif
943
944#if defined(HASHED_PAGE_VIRTUAL)
Ian Campbellf9918792011-08-17 13:45:09 +0100945void *page_address(const struct page *page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700946void set_page_address(struct page *page, void *virtual);
947void page_address_init(void);
948#endif
949
950#if !defined(HASHED_PAGE_VIRTUAL) && !defined(WANT_PAGE_VIRTUAL)
951#define page_address(page) lowmem_page_address(page)
952#define set_page_address(page, address) do { } while(0)
953#define page_address_init() do { } while(0)
954#endif
955
Kirill A. Shutemove39155e2015-04-15 16:14:53 -0700956extern void *page_rmapping(struct page *page);
957extern struct anon_vma *page_anon_vma(struct page *page);
Shaohua Li98003392013-02-22 16:34:35 -0800958extern struct address_space *page_mapping(struct page *page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959
Mel Gormanf981c592012-07-31 16:44:47 -0700960extern struct address_space *__page_file_mapping(struct page *);
961
962static inline
963struct address_space *page_file_mapping(struct page *page)
964{
965 if (unlikely(PageSwapCache(page)))
966 return __page_file_mapping(page);
967
968 return page->mapping;
969}
970
Linus Torvalds1da177e2005-04-16 15:20:36 -0700971/*
972 * Return the pagecache index of the passed page. Regular pagecache pages
973 * use ->index whereas swapcache pages use ->private
974 */
975static inline pgoff_t page_index(struct page *page)
976{
977 if (unlikely(PageSwapCache(page)))
Hugh Dickins4c21e2f2005-10-29 18:16:40 -0700978 return page_private(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979 return page->index;
980}
981
Mel Gormanf981c592012-07-31 16:44:47 -0700982extern pgoff_t __page_file_index(struct page *page);
983
984/*
985 * Return the file index of the page. Regular pagecache pages use ->index
986 * whereas swapcache pages use swp_offset(->private)
987 */
988static inline pgoff_t page_file_index(struct page *page)
989{
990 if (unlikely(PageSwapCache(page)))
991 return __page_file_index(page);
992
993 return page->index;
994}
995
Linus Torvalds1da177e2005-04-16 15:20:36 -0700996/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700997 * Return true if this page is mapped into pagetables.
998 */
999static inline int page_mapped(struct page *page)
1000{
1001 return atomic_read(&(page)->_mapcount) >= 0;
1002}
1003
1004/*
Linus Torvalds1da177e2005-04-16 15:20:36 -07001005 * Different kinds of faults, as returned by handle_mm_fault().
1006 * Used to decide whether a process gets delivered SIGBUS or
1007 * just gets major/minor fault counters bumped up.
1008 */
Nick Piggind0217ac2007-07-19 01:47:03 -07001009
Nick Piggin83c54072007-07-19 01:47:05 -07001010#define VM_FAULT_MINOR 0 /* For backwards compat. Remove me quickly. */
Nick Piggind0217ac2007-07-19 01:47:03 -07001011
Nick Piggin83c54072007-07-19 01:47:05 -07001012#define VM_FAULT_OOM 0x0001
1013#define VM_FAULT_SIGBUS 0x0002
1014#define VM_FAULT_MAJOR 0x0004
1015#define VM_FAULT_WRITE 0x0008 /* Special case for get_user_pages */
Andi Kleenaa50d3a2010-10-06 21:45:00 +02001016#define VM_FAULT_HWPOISON 0x0010 /* Hit poisoned small page */
1017#define VM_FAULT_HWPOISON_LARGE 0x0020 /* Hit poisoned large page. Index encoded in upper bits */
Linus Torvalds33692f22015-01-29 10:51:32 -08001018#define VM_FAULT_SIGSEGV 0x0040
Nick Pigginf33ea7f2005-08-03 20:24:01 +10001019
Nick Piggin83c54072007-07-19 01:47:05 -07001020#define VM_FAULT_NOPAGE 0x0100 /* ->fault installed the pte, not return page */
1021#define VM_FAULT_LOCKED 0x0200 /* ->fault locked the returned page */
Michel Lespinassed065bd82010-10-26 14:21:57 -07001022#define VM_FAULT_RETRY 0x0400 /* ->fault blocked, must retry */
Kirill A. Shutemovc0292552013-09-12 15:14:05 -07001023#define VM_FAULT_FALLBACK 0x0800 /* huge page fault failed, fall back to small */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001024
Andi Kleenaa50d3a2010-10-06 21:45:00 +02001025#define VM_FAULT_HWPOISON_LARGE_MASK 0xf000 /* encodes hpage index for large hwpoison */
1026
Linus Torvalds33692f22015-01-29 10:51:32 -08001027#define VM_FAULT_ERROR (VM_FAULT_OOM | VM_FAULT_SIGBUS | VM_FAULT_SIGSEGV | \
1028 VM_FAULT_HWPOISON | VM_FAULT_HWPOISON_LARGE | \
1029 VM_FAULT_FALLBACK)
Andi Kleenaa50d3a2010-10-06 21:45:00 +02001030
1031/* Encode hstate index for a hwpoisoned large page */
1032#define VM_FAULT_SET_HINDEX(x) ((x) << 12)
1033#define VM_FAULT_GET_HINDEX(x) (((x) >> 12) & 0xf)
Nick Piggind0217ac2007-07-19 01:47:03 -07001034
Nick Piggin1c0fe6e2009-01-06 14:38:59 -08001035/*
1036 * Can be called by the pagefault handler when it gets a VM_FAULT_OOM.
1037 */
1038extern void pagefault_out_of_memory(void);
1039
Linus Torvalds1da177e2005-04-16 15:20:36 -07001040#define offset_in_page(p) ((unsigned long)(p) & ~PAGE_MASK)
1041
David Rientjesddd588b2011-03-22 16:30:46 -07001042/*
David Rientjes7bf02ea2011-05-24 17:11:16 -07001043 * Flags passed to show_mem() and show_free_areas() to suppress output in
David Rientjesddd588b2011-03-22 16:30:46 -07001044 * various contexts.
1045 */
David Rientjes4b59e6c2013-04-29 15:06:11 -07001046#define SHOW_MEM_FILTER_NODES (0x0001u) /* disallowed nodes */
David Rientjesddd588b2011-03-22 16:30:46 -07001047
David Rientjes7bf02ea2011-05-24 17:11:16 -07001048extern void show_free_areas(unsigned int flags);
1049extern bool skip_free_areas_node(unsigned int flags, int nid);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001050
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051int shmem_zero_setup(struct vm_area_struct *);
Johannes Weiner0cd61442014-04-03 14:47:46 -07001052#ifdef CONFIG_SHMEM
1053bool shmem_mapping(struct address_space *mapping);
1054#else
1055static inline bool shmem_mapping(struct address_space *mapping)
1056{
1057 return false;
1058}
1059#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001060
Alexey Dobriyane8edc6e2007-05-21 01:22:52 +04001061extern int can_do_mlock(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001062extern int user_shm_lock(size_t, struct user_struct *);
1063extern void user_shm_unlock(size_t, struct user_struct *);
1064
1065/*
1066 * Parameter block passed down to zap_pte_range in exceptional cases.
1067 */
1068struct zap_details {
Linus Torvalds1da177e2005-04-16 15:20:36 -07001069 struct address_space *check_mapping; /* Check page->mapping if set */
1070 pgoff_t first_index; /* Lowest page->index to unmap */
1071 pgoff_t last_index; /* Highest page->index to unmap */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001072};
1073
Nick Piggin7e675132008-04-28 02:13:00 -07001074struct page *vm_normal_page(struct vm_area_struct *vma, unsigned long addr,
1075 pte_t pte);
1076
Jack Steinerc627f9c2008-07-29 22:33:53 -07001077int zap_vma_ptes(struct vm_area_struct *vma, unsigned long address,
1078 unsigned long size);
Al Viro14f5ff52012-03-05 13:38:09 -05001079void zap_page_range(struct vm_area_struct *vma, unsigned long address,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001080 unsigned long size, struct zap_details *);
Linus Torvalds4f74d2c2012-05-06 13:54:06 -07001081void unmap_vmas(struct mmu_gather *tlb, struct vm_area_struct *start_vma,
1082 unsigned long start, unsigned long end);
Matt Mackalle6473092008-02-04 22:29:01 -08001083
1084/**
1085 * mm_walk - callbacks for walk_page_range
Matt Mackalle6473092008-02-04 22:29:01 -08001086 * @pmd_entry: if set, called for each non-empty PMD (3rd-level) entry
Dave Hansen03319322011-03-22 16:32:56 -07001087 * this handler is required to be able to handle
1088 * pmd_trans_huge() pmds. They may simply choose to
1089 * split_huge_page() instead of handling it explicitly.
Matt Mackalle6473092008-02-04 22:29:01 -08001090 * @pte_entry: if set, called for each non-empty PTE (4th-level) entry
1091 * @pte_hole: if set, called for each hole at all levels
Naoya Horiguchi5dc37642009-12-14 18:00:01 -08001092 * @hugetlb_entry: if set, called for each hugetlb entry
Naoya Horiguchifafaa422015-02-11 15:27:37 -08001093 * @test_walk: caller specific callback function to determine whether
1094 * we walk over the current vma or not. A positive returned
1095 * value means "do page table walk over the current vma,"
1096 * and a negative one means "abort current page table walk
1097 * right now." 0 means "skip the current vma."
1098 * @mm: mm_struct representing the target process of page table walk
1099 * @vma: vma currently walked (NULL if walking outside vmas)
1100 * @private: private data for callbacks' usage
Matt Mackalle6473092008-02-04 22:29:01 -08001101 *
Naoya Horiguchifafaa422015-02-11 15:27:37 -08001102 * (see the comment on walk_page_range() for more details)
Matt Mackalle6473092008-02-04 22:29:01 -08001103 */
1104struct mm_walk {
Andrew Morton0f157a52013-05-07 16:18:10 -07001105 int (*pmd_entry)(pmd_t *pmd, unsigned long addr,
1106 unsigned long next, struct mm_walk *walk);
1107 int (*pte_entry)(pte_t *pte, unsigned long addr,
1108 unsigned long next, struct mm_walk *walk);
1109 int (*pte_hole)(unsigned long addr, unsigned long next,
1110 struct mm_walk *walk);
1111 int (*hugetlb_entry)(pte_t *pte, unsigned long hmask,
1112 unsigned long addr, unsigned long next,
1113 struct mm_walk *walk);
Naoya Horiguchifafaa422015-02-11 15:27:37 -08001114 int (*test_walk)(unsigned long addr, unsigned long next,
1115 struct mm_walk *walk);
Dave Hansen21650092008-06-12 15:21:47 -07001116 struct mm_struct *mm;
Naoya Horiguchifafaa422015-02-11 15:27:37 -08001117 struct vm_area_struct *vma;
Dave Hansen21650092008-06-12 15:21:47 -07001118 void *private;
Matt Mackalle6473092008-02-04 22:29:01 -08001119};
1120
Dave Hansen21650092008-06-12 15:21:47 -07001121int walk_page_range(unsigned long addr, unsigned long end,
1122 struct mm_walk *walk);
Naoya Horiguchi900fc5f2015-02-11 15:27:40 -08001123int walk_page_vma(struct vm_area_struct *vma, struct mm_walk *walk);
Jan Beulich42b77722008-07-23 21:27:10 -07001124void free_pgd_range(struct mmu_gather *tlb, unsigned long addr,
Hugh Dickins3bf5ee92005-04-19 13:29:16 -07001125 unsigned long end, unsigned long floor, unsigned long ceiling);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001126int copy_page_range(struct mm_struct *dst, struct mm_struct *src,
1127 struct vm_area_struct *vma);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001128void unmap_mapping_range(struct address_space *mapping,
1129 loff_t const holebegin, loff_t const holelen, int even_cows);
Johannes Weiner3b6748e2009-06-16 15:32:35 -07001130int follow_pfn(struct vm_area_struct *vma, unsigned long address,
1131 unsigned long *pfn);
venkatesh.pallipadi@intel.comd87fe662008-12-19 13:47:27 -08001132int follow_phys(struct vm_area_struct *vma, unsigned long address,
1133 unsigned int flags, unsigned long *prot, resource_size_t *phys);
Rik van Riel28b2ee22008-07-23 21:27:05 -07001134int generic_access_phys(struct vm_area_struct *vma, unsigned long addr,
1135 void *buf, int len, int write);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001136
1137static inline void unmap_shared_mapping_range(struct address_space *mapping,
1138 loff_t const holebegin, loff_t const holelen)
1139{
1140 unmap_mapping_range(mapping, holebegin, holelen, 0);
1141}
1142
Kirill A. Shutemov7caef262013-09-12 15:13:56 -07001143extern void truncate_pagecache(struct inode *inode, loff_t new);
Christoph Hellwig2c27c652010-06-04 11:30:04 +02001144extern void truncate_setsize(struct inode *inode, loff_t newsize);
Jan Kara90a80202014-10-01 21:49:18 -04001145void pagecache_isize_extended(struct inode *inode, loff_t from, loff_t to);
Hugh Dickins623e3db2012-03-28 14:42:40 -07001146void truncate_pagecache_range(struct inode *inode, loff_t offset, loff_t end);
Nick Piggin750b4982009-09-16 11:50:12 +02001147int truncate_inode_page(struct address_space *mapping, struct page *page);
Andi Kleen25718732009-09-16 11:50:13 +02001148int generic_error_remove_page(struct address_space *mapping, struct page *page);
Wu Fengguang83f78662009-09-16 11:50:13 +02001149int invalidate_inode_page(struct page *page);
1150
David Howells7ee1dd32006-01-06 00:11:44 -08001151#ifdef CONFIG_MMU
Nick Piggin83c54072007-07-19 01:47:05 -07001152extern int handle_mm_fault(struct mm_struct *mm, struct vm_area_struct *vma,
Linus Torvaldsd06063c2009-04-10 09:01:23 -07001153 unsigned long address, unsigned int flags);
Peter Zijlstra5c723ba2011-07-27 12:17:11 +02001154extern int fixup_user_fault(struct task_struct *tsk, struct mm_struct *mm,
1155 unsigned long address, unsigned int fault_flags);
David Howells7ee1dd32006-01-06 00:11:44 -08001156#else
1157static inline int handle_mm_fault(struct mm_struct *mm,
1158 struct vm_area_struct *vma, unsigned long address,
Linus Torvaldsd06063c2009-04-10 09:01:23 -07001159 unsigned int flags)
David Howells7ee1dd32006-01-06 00:11:44 -08001160{
1161 /* should never happen if there's no MMU */
1162 BUG();
1163 return VM_FAULT_SIGBUS;
1164}
Peter Zijlstra5c723ba2011-07-27 12:17:11 +02001165static inline int fixup_user_fault(struct task_struct *tsk,
1166 struct mm_struct *mm, unsigned long address,
1167 unsigned int fault_flags)
1168{
1169 /* should never happen if there's no MMU */
1170 BUG();
1171 return -EFAULT;
1172}
David Howells7ee1dd32006-01-06 00:11:44 -08001173#endif
Nick Pigginf33ea7f2005-08-03 20:24:01 +10001174
Linus Torvalds1da177e2005-04-16 15:20:36 -07001175extern int access_process_vm(struct task_struct *tsk, unsigned long addr, void *buf, int len, int write);
Stephen Wilson5ddd36b2011-03-13 15:49:20 -04001176extern int access_remote_vm(struct mm_struct *mm, unsigned long addr,
1177 void *buf, int len, int write);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001178
Michel Lespinasse28a35712013-02-22 16:35:55 -08001179long __get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
1180 unsigned long start, unsigned long nr_pages,
1181 unsigned int foll_flags, struct page **pages,
1182 struct vm_area_struct **vmas, int *nonblocking);
1183long get_user_pages(struct task_struct *tsk, struct mm_struct *mm,
1184 unsigned long start, unsigned long nr_pages,
1185 int write, int force, struct page **pages,
1186 struct vm_area_struct **vmas);
Andrea Arcangelif0818f42015-02-11 15:27:17 -08001187long get_user_pages_locked(struct task_struct *tsk, struct mm_struct *mm,
1188 unsigned long start, unsigned long nr_pages,
1189 int write, int force, struct page **pages,
1190 int *locked);
Andrea Arcangeli0fd71a52015-02-11 15:27:20 -08001191long __get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
1192 unsigned long start, unsigned long nr_pages,
1193 int write, int force, struct page **pages,
1194 unsigned int gup_flags);
Andrea Arcangelif0818f42015-02-11 15:27:17 -08001195long get_user_pages_unlocked(struct task_struct *tsk, struct mm_struct *mm,
1196 unsigned long start, unsigned long nr_pages,
1197 int write, int force, struct page **pages);
Nick Piggind2bf6be2009-06-16 15:31:39 -07001198int get_user_pages_fast(unsigned long start, int nr_pages, int write,
1199 struct page **pages);
Mel Gorman18022c52012-07-31 16:44:51 -07001200struct kvec;
1201int get_kernel_pages(const struct kvec *iov, int nr_pages, int write,
1202 struct page **pages);
1203int get_kernel_page(unsigned long start, int write, struct page **pages);
Hugh Dickinsf3e8fcc2009-09-21 17:03:25 -07001204struct page *get_dump_page(unsigned long addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001205
David Howellscf9a2ae2006-08-29 19:05:54 +01001206extern int try_to_release_page(struct page * page, gfp_t gfp_mask);
Lukas Czernerd47992f2013-05-21 23:17:23 -04001207extern void do_invalidatepage(struct page *page, unsigned int offset,
1208 unsigned int length);
David Howellscf9a2ae2006-08-29 19:05:54 +01001209
Linus Torvalds1da177e2005-04-16 15:20:36 -07001210int __set_page_dirty_nobuffers(struct page *page);
Ken Chen76719322007-02-10 01:43:15 -08001211int __set_page_dirty_no_writeback(struct page *page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001212int redirty_page_for_writepage(struct writeback_control *wbc,
1213 struct page *page);
Edward Shishkine3a7cca2009-03-31 15:19:39 -07001214void account_page_dirtied(struct page *page, struct address_space *mapping);
Konstantin Khlebnikovb9ea2512015-04-14 15:45:27 -07001215void account_page_cleaned(struct page *page, struct address_space *mapping);
Harvey Harrisonb3c97522008-02-13 15:03:15 -08001216int set_page_dirty(struct page *page);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001217int set_page_dirty_lock(struct page *page);
1218int clear_page_dirty_for_io(struct page *page);
Konstantin Khlebnikovb9ea2512015-04-14 15:45:27 -07001219
William Robertsa9090252014-02-11 10:11:59 -08001220int get_cmdline(struct task_struct *task, char *buffer, int buflen);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001221
Stefan Bader39aa3cb2010-08-31 15:52:27 +02001222/* Is the vma a continuation of the stack vma above it? */
Mikulas Patockaa09a79f2011-05-09 13:01:09 +02001223static inline int vma_growsdown(struct vm_area_struct *vma, unsigned long addr)
Stefan Bader39aa3cb2010-08-31 15:52:27 +02001224{
1225 return vma && (vma->vm_end == addr) && (vma->vm_flags & VM_GROWSDOWN);
1226}
1227
Mikulas Patockaa09a79f2011-05-09 13:01:09 +02001228static inline int stack_guard_page_start(struct vm_area_struct *vma,
1229 unsigned long addr)
1230{
1231 return (vma->vm_flags & VM_GROWSDOWN) &&
1232 (vma->vm_start == addr) &&
1233 !vma_growsdown(vma->vm_prev, addr);
1234}
1235
1236/* Is the vma a continuation of the stack vma below it? */
1237static inline int vma_growsup(struct vm_area_struct *vma, unsigned long addr)
1238{
1239 return vma && (vma->vm_start == addr) && (vma->vm_flags & VM_GROWSUP);
1240}
1241
1242static inline int stack_guard_page_end(struct vm_area_struct *vma,
1243 unsigned long addr)
1244{
1245 return (vma->vm_flags & VM_GROWSUP) &&
1246 (vma->vm_end == addr) &&
1247 !vma_growsup(vma->vm_next, addr);
1248}
1249
Oleg Nesterov58cb6542014-10-09 15:25:54 -07001250extern struct task_struct *task_of_stack(struct task_struct *task,
1251 struct vm_area_struct *vma, bool in_group);
Siddhesh Poyarekarb7643752012-03-21 16:34:04 -07001252
Ollie Wildb6a2fea2007-07-19 01:48:16 -07001253extern unsigned long move_page_tables(struct vm_area_struct *vma,
1254 unsigned long old_addr, struct vm_area_struct *new_vma,
Michel Lespinasse38a76012012-10-08 16:31:50 -07001255 unsigned long new_addr, unsigned long len,
1256 bool need_rmap_locks);
Peter Zijlstra7da4d642012-11-19 03:14:23 +01001257extern unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
1258 unsigned long end, pgprot_t newprot,
Mel Gorman4b10e7d2012-10-25 14:16:32 +02001259 int dirty_accountable, int prot_numa);
Ollie Wildb6a2fea2007-07-19 01:48:16 -07001260extern int mprotect_fixup(struct vm_area_struct *vma,
1261 struct vm_area_struct **pprev, unsigned long start,
1262 unsigned long end, unsigned long newflags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001263
Nick Piggin21cc1992008-07-25 19:45:22 -07001264/*
Peter Zijlstra465a4542009-06-15 12:31:37 +02001265 * doesn't attempt to fault and will return short.
1266 */
1267int __get_user_pages_fast(unsigned long start, int nr_pages, int write,
1268 struct page **pages);
KAMEZAWA Hiroyukid559db02010-03-05 13:41:39 -08001269/*
1270 * per-process(per-mm_struct) statistics.
1271 */
Matt Fleming172703b2011-05-24 17:12:36 -07001272static inline unsigned long get_mm_counter(struct mm_struct *mm, int member)
1273{
Konstantin Khlebnikov69c97822012-03-21 16:33:49 -07001274 long val = atomic_long_read(&mm->rss_stat.count[member]);
1275
1276#ifdef SPLIT_RSS_COUNTING
1277 /*
1278 * counter is updated in asynchronous manner and may go to minus.
1279 * But it's never be expected number for users.
1280 */
1281 if (val < 0)
1282 val = 0;
Matt Fleming172703b2011-05-24 17:12:36 -07001283#endif
Konstantin Khlebnikov69c97822012-03-21 16:33:49 -07001284 return (unsigned long)val;
1285}
KAMEZAWA Hiroyukid559db02010-03-05 13:41:39 -08001286
1287static inline void add_mm_counter(struct mm_struct *mm, int member, long value)
1288{
1289 atomic_long_add(value, &mm->rss_stat.count[member]);
1290}
1291
1292static inline void inc_mm_counter(struct mm_struct *mm, int member)
1293{
1294 atomic_long_inc(&mm->rss_stat.count[member]);
1295}
1296
1297static inline void dec_mm_counter(struct mm_struct *mm, int member)
1298{
1299 atomic_long_dec(&mm->rss_stat.count[member]);
1300}
1301
KAMEZAWA Hiroyukid559db02010-03-05 13:41:39 -08001302static inline unsigned long get_mm_rss(struct mm_struct *mm)
1303{
1304 return get_mm_counter(mm, MM_FILEPAGES) +
1305 get_mm_counter(mm, MM_ANONPAGES);
1306}
1307
1308static inline unsigned long get_mm_hiwater_rss(struct mm_struct *mm)
1309{
1310 return max(mm->hiwater_rss, get_mm_rss(mm));
1311}
1312
1313static inline unsigned long get_mm_hiwater_vm(struct mm_struct *mm)
1314{
1315 return max(mm->hiwater_vm, mm->total_vm);
1316}
1317
1318static inline void update_hiwater_rss(struct mm_struct *mm)
1319{
1320 unsigned long _rss = get_mm_rss(mm);
1321
1322 if ((mm)->hiwater_rss < _rss)
1323 (mm)->hiwater_rss = _rss;
1324}
1325
1326static inline void update_hiwater_vm(struct mm_struct *mm)
1327{
1328 if (mm->hiwater_vm < mm->total_vm)
1329 mm->hiwater_vm = mm->total_vm;
1330}
1331
Petr Cermak695f0552015-02-12 15:01:00 -08001332static inline void reset_mm_hiwater_rss(struct mm_struct *mm)
1333{
1334 mm->hiwater_rss = get_mm_rss(mm);
1335}
1336
KAMEZAWA Hiroyukid559db02010-03-05 13:41:39 -08001337static inline void setmax_mm_hiwater_rss(unsigned long *maxrss,
1338 struct mm_struct *mm)
1339{
1340 unsigned long hiwater_rss = get_mm_hiwater_rss(mm);
1341
1342 if (*maxrss < hiwater_rss)
1343 *maxrss = hiwater_rss;
1344}
1345
KAMEZAWA Hiroyuki53bddb42010-03-10 15:20:38 -08001346#if defined(SPLIT_RSS_COUNTING)
David Rientjes05af2e12012-03-21 16:34:13 -07001347void sync_mm_rss(struct mm_struct *mm);
KAMEZAWA Hiroyuki53bddb42010-03-10 15:20:38 -08001348#else
David Rientjes05af2e12012-03-21 16:34:13 -07001349static inline void sync_mm_rss(struct mm_struct *mm)
KAMEZAWA Hiroyuki53bddb42010-03-10 15:20:38 -08001350{
1351}
1352#endif
Peter Zijlstra465a4542009-06-15 12:31:37 +02001353
Alexey Dobriyan4e950f62007-07-30 02:36:13 +04001354int vma_wants_writenotify(struct vm_area_struct *vma);
Peter Zijlstrad08b3852006-09-25 23:30:57 -07001355
Namhyung Kim25ca1d62010-10-26 14:21:59 -07001356extern pte_t *__get_locked_pte(struct mm_struct *mm, unsigned long addr,
1357 spinlock_t **ptl);
1358static inline pte_t *get_locked_pte(struct mm_struct *mm, unsigned long addr,
1359 spinlock_t **ptl)
1360{
1361 pte_t *ptep;
1362 __cond_lock(*ptl, ptep = __get_locked_pte(mm, addr, ptl));
1363 return ptep;
1364}
Linus Torvaldsc9cfcdd2005-11-29 14:03:14 -08001365
Nick Piggin5f22df02007-05-06 14:49:02 -07001366#ifdef __PAGETABLE_PUD_FOLDED
1367static inline int __pud_alloc(struct mm_struct *mm, pgd_t *pgd,
1368 unsigned long address)
1369{
1370 return 0;
1371}
1372#else
Hugh Dickins1bb36302005-10-29 18:16:22 -07001373int __pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address);
Nick Piggin5f22df02007-05-06 14:49:02 -07001374#endif
1375
Kirill A. Shutemov2d2f5112015-02-12 14:59:59 -08001376#if defined(__PAGETABLE_PMD_FOLDED) || !defined(CONFIG_MMU)
Nick Piggin5f22df02007-05-06 14:49:02 -07001377static inline int __pmd_alloc(struct mm_struct *mm, pud_t *pud,
1378 unsigned long address)
1379{
1380 return 0;
1381}
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -08001382
Kirill A. Shutemov2d2f5112015-02-12 14:59:59 -08001383static inline void mm_nr_pmds_init(struct mm_struct *mm) {}
1384
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -08001385static inline unsigned long mm_nr_pmds(struct mm_struct *mm)
1386{
1387 return 0;
1388}
1389
1390static inline void mm_inc_nr_pmds(struct mm_struct *mm) {}
1391static inline void mm_dec_nr_pmds(struct mm_struct *mm) {}
1392
Nick Piggin5f22df02007-05-06 14:49:02 -07001393#else
Hugh Dickins1bb36302005-10-29 18:16:22 -07001394int __pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address);
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -08001395
Kirill A. Shutemov2d2f5112015-02-12 14:59:59 -08001396static inline void mm_nr_pmds_init(struct mm_struct *mm)
1397{
1398 atomic_long_set(&mm->nr_pmds, 0);
1399}
1400
Kirill A. Shutemovdc6c9a32015-02-11 15:26:50 -08001401static inline unsigned long mm_nr_pmds(struct mm_struct *mm)
1402{
1403 return atomic_long_read(&mm->nr_pmds);
1404}
1405
1406static inline void mm_inc_nr_pmds(struct mm_struct *mm)
1407{
1408 atomic_long_inc(&mm->nr_pmds);
1409}
1410
1411static inline void mm_dec_nr_pmds(struct mm_struct *mm)
1412{
1413 atomic_long_dec(&mm->nr_pmds);
1414}
Nick Piggin5f22df02007-05-06 14:49:02 -07001415#endif
1416
Andrea Arcangeli8ac1f832011-01-13 15:46:43 -08001417int __pte_alloc(struct mm_struct *mm, struct vm_area_struct *vma,
1418 pmd_t *pmd, unsigned long address);
Hugh Dickins1bb36302005-10-29 18:16:22 -07001419int __pte_alloc_kernel(pmd_t *pmd, unsigned long address);
1420
Linus Torvalds1da177e2005-04-16 15:20:36 -07001421/*
1422 * The following ifdef needed to get the 4level-fixup.h header to work.
1423 * Remove it when 4level-fixup.h has been removed.
1424 */
Hugh Dickins1bb36302005-10-29 18:16:22 -07001425#if defined(CONFIG_MMU) && !defined(__ARCH_HAS_4LEVEL_HACK)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001426static inline pud_t *pud_alloc(struct mm_struct *mm, pgd_t *pgd, unsigned long address)
1427{
Hugh Dickins1bb36302005-10-29 18:16:22 -07001428 return (unlikely(pgd_none(*pgd)) && __pud_alloc(mm, pgd, address))?
1429 NULL: pud_offset(pgd, address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001430}
1431
1432static inline pmd_t *pmd_alloc(struct mm_struct *mm, pud_t *pud, unsigned long address)
1433{
Hugh Dickins1bb36302005-10-29 18:16:22 -07001434 return (unlikely(pud_none(*pud)) && __pmd_alloc(mm, pud, address))?
1435 NULL: pmd_offset(pud, address);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001436}
Hugh Dickins1bb36302005-10-29 18:16:22 -07001437#endif /* CONFIG_MMU && !__ARCH_HAS_4LEVEL_HACK */
1438
Kirill A. Shutemov57c1ffc2013-11-14 14:30:45 -08001439#if USE_SPLIT_PTE_PTLOCKS
Kirill A. Shutemov597d7952013-12-20 13:35:58 +02001440#if ALLOC_SPLIT_PTLOCKS
Kirill A. Shutemovb35f1812014-01-21 15:49:07 -08001441void __init ptlock_cache_init(void);
Peter Zijlstra539edb52013-11-14 14:31:52 -08001442extern bool ptlock_alloc(struct page *page);
1443extern void ptlock_free(struct page *page);
1444
1445static inline spinlock_t *ptlock_ptr(struct page *page)
1446{
1447 return page->ptl;
1448}
Kirill A. Shutemov597d7952013-12-20 13:35:58 +02001449#else /* ALLOC_SPLIT_PTLOCKS */
Kirill A. Shutemovb35f1812014-01-21 15:49:07 -08001450static inline void ptlock_cache_init(void)
1451{
1452}
1453
Kirill A. Shutemov49076ec2013-11-14 14:31:51 -08001454static inline bool ptlock_alloc(struct page *page)
1455{
Kirill A. Shutemov49076ec2013-11-14 14:31:51 -08001456 return true;
1457}
Peter Zijlstra539edb52013-11-14 14:31:52 -08001458
Kirill A. Shutemov49076ec2013-11-14 14:31:51 -08001459static inline void ptlock_free(struct page *page)
1460{
Kirill A. Shutemov49076ec2013-11-14 14:31:51 -08001461}
1462
1463static inline spinlock_t *ptlock_ptr(struct page *page)
1464{
Peter Zijlstra539edb52013-11-14 14:31:52 -08001465 return &page->ptl;
Kirill A. Shutemov49076ec2013-11-14 14:31:51 -08001466}
Kirill A. Shutemov597d7952013-12-20 13:35:58 +02001467#endif /* ALLOC_SPLIT_PTLOCKS */
Kirill A. Shutemov49076ec2013-11-14 14:31:51 -08001468
1469static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1470{
1471 return ptlock_ptr(pmd_page(*pmd));
1472}
1473
1474static inline bool ptlock_init(struct page *page)
1475{
1476 /*
1477 * prep_new_page() initialize page->private (and therefore page->ptl)
1478 * with 0. Make sure nobody took it in use in between.
1479 *
1480 * It can happen if arch try to use slab for page table allocation:
1481 * slab code uses page->slab_cache and page->first_page (for tail
1482 * pages), which share storage with page->ptl.
1483 */
Sasha Levin309381fea2014-01-23 15:52:54 -08001484 VM_BUG_ON_PAGE(*(unsigned long *)&page->ptl, page);
Kirill A. Shutemov49076ec2013-11-14 14:31:51 -08001485 if (!ptlock_alloc(page))
1486 return false;
1487 spin_lock_init(ptlock_ptr(page));
1488 return true;
1489}
1490
1491/* Reset page->mapping so free_pages_check won't complain. */
1492static inline void pte_lock_deinit(struct page *page)
1493{
1494 page->mapping = NULL;
1495 ptlock_free(page);
1496}
1497
Kirill A. Shutemov57c1ffc2013-11-14 14:30:45 -08001498#else /* !USE_SPLIT_PTE_PTLOCKS */
Hugh Dickins4c21e2f2005-10-29 18:16:40 -07001499/*
1500 * We use mm->page_table_lock to guard all pagetable pages of the mm.
1501 */
Kirill A. Shutemov49076ec2013-11-14 14:31:51 -08001502static inline spinlock_t *pte_lockptr(struct mm_struct *mm, pmd_t *pmd)
1503{
1504 return &mm->page_table_lock;
1505}
Kirill A. Shutemovb35f1812014-01-21 15:49:07 -08001506static inline void ptlock_cache_init(void) {}
Kirill A. Shutemov49076ec2013-11-14 14:31:51 -08001507static inline bool ptlock_init(struct page *page) { return true; }
1508static inline void pte_lock_deinit(struct page *page) {}
Kirill A. Shutemov57c1ffc2013-11-14 14:30:45 -08001509#endif /* USE_SPLIT_PTE_PTLOCKS */
Hugh Dickins4c21e2f2005-10-29 18:16:40 -07001510
Kirill A. Shutemovb35f1812014-01-21 15:49:07 -08001511static inline void pgtable_init(void)
1512{
1513 ptlock_cache_init();
1514 pgtable_cache_init();
1515}
1516
Kirill A. Shutemov390f44e2013-11-14 14:31:20 -08001517static inline bool pgtable_page_ctor(struct page *page)
Martin Schwidefsky2f569af2008-02-08 04:22:04 -08001518{
Martin Schwidefsky2f569af2008-02-08 04:22:04 -08001519 inc_zone_page_state(page, NR_PAGETABLE);
Kirill A. Shutemov49076ec2013-11-14 14:31:51 -08001520 return ptlock_init(page);
Martin Schwidefsky2f569af2008-02-08 04:22:04 -08001521}
1522
1523static inline void pgtable_page_dtor(struct page *page)
1524{
1525 pte_lock_deinit(page);
1526 dec_zone_page_state(page, NR_PAGETABLE);
1527}
1528
Hugh Dickinsc74df322005-10-29 18:16:23 -07001529#define pte_offset_map_lock(mm, pmd, address, ptlp) \
1530({ \
Hugh Dickins4c21e2f2005-10-29 18:16:40 -07001531 spinlock_t *__ptl = pte_lockptr(mm, pmd); \
Hugh Dickinsc74df322005-10-29 18:16:23 -07001532 pte_t *__pte = pte_offset_map(pmd, address); \
1533 *(ptlp) = __ptl; \
1534 spin_lock(__ptl); \
1535 __pte; \
1536})
1537
1538#define pte_unmap_unlock(pte, ptl) do { \
1539 spin_unlock(ptl); \
1540 pte_unmap(pte); \
1541} while (0)
1542
Andrea Arcangeli8ac1f832011-01-13 15:46:43 -08001543#define pte_alloc_map(mm, vma, pmd, address) \
1544 ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, vma, \
1545 pmd, address))? \
1546 NULL: pte_offset_map(pmd, address))
Hugh Dickins1bb36302005-10-29 18:16:22 -07001547
Hugh Dickinsc74df322005-10-29 18:16:23 -07001548#define pte_alloc_map_lock(mm, pmd, address, ptlp) \
Andrea Arcangeli8ac1f832011-01-13 15:46:43 -08001549 ((unlikely(pmd_none(*(pmd))) && __pte_alloc(mm, NULL, \
1550 pmd, address))? \
Hugh Dickinsc74df322005-10-29 18:16:23 -07001551 NULL: pte_offset_map_lock(mm, pmd, address, ptlp))
1552
Hugh Dickins1bb36302005-10-29 18:16:22 -07001553#define pte_alloc_kernel(pmd, address) \
Andrea Arcangeli8ac1f832011-01-13 15:46:43 -08001554 ((unlikely(pmd_none(*(pmd))) && __pte_alloc_kernel(pmd, address))? \
Hugh Dickins1bb36302005-10-29 18:16:22 -07001555 NULL: pte_offset_kernel(pmd, address))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001556
Kirill A. Shutemove009bb32013-11-14 14:31:07 -08001557#if USE_SPLIT_PMD_PTLOCKS
1558
Martin Schwidefsky634391a2014-02-13 13:53:33 +01001559static struct page *pmd_to_page(pmd_t *pmd)
1560{
1561 unsigned long mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
1562 return virt_to_page((void *)((unsigned long) pmd & mask));
1563}
1564
Kirill A. Shutemove009bb32013-11-14 14:31:07 -08001565static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1566{
Martin Schwidefsky634391a2014-02-13 13:53:33 +01001567 return ptlock_ptr(pmd_to_page(pmd));
Kirill A. Shutemove009bb32013-11-14 14:31:07 -08001568}
1569
1570static inline bool pgtable_pmd_page_ctor(struct page *page)
1571{
Kirill A. Shutemove009bb32013-11-14 14:31:07 -08001572#ifdef CONFIG_TRANSPARENT_HUGEPAGE
1573 page->pmd_huge_pte = NULL;
1574#endif
Kirill A. Shutemov49076ec2013-11-14 14:31:51 -08001575 return ptlock_init(page);
Kirill A. Shutemove009bb32013-11-14 14:31:07 -08001576}
1577
1578static inline void pgtable_pmd_page_dtor(struct page *page)
1579{
1580#ifdef CONFIG_TRANSPARENT_HUGEPAGE
Sasha Levin309381fea2014-01-23 15:52:54 -08001581 VM_BUG_ON_PAGE(page->pmd_huge_pte, page);
Kirill A. Shutemove009bb32013-11-14 14:31:07 -08001582#endif
Kirill A. Shutemov49076ec2013-11-14 14:31:51 -08001583 ptlock_free(page);
Kirill A. Shutemove009bb32013-11-14 14:31:07 -08001584}
1585
Martin Schwidefsky634391a2014-02-13 13:53:33 +01001586#define pmd_huge_pte(mm, pmd) (pmd_to_page(pmd)->pmd_huge_pte)
Kirill A. Shutemove009bb32013-11-14 14:31:07 -08001587
1588#else
1589
Kirill A. Shutemov9a86cb72013-11-14 14:30:51 -08001590static inline spinlock_t *pmd_lockptr(struct mm_struct *mm, pmd_t *pmd)
1591{
1592 return &mm->page_table_lock;
1593}
1594
Kirill A. Shutemove009bb32013-11-14 14:31:07 -08001595static inline bool pgtable_pmd_page_ctor(struct page *page) { return true; }
1596static inline void pgtable_pmd_page_dtor(struct page *page) {}
1597
Kirill A. Shutemovc389a252013-11-14 14:30:59 -08001598#define pmd_huge_pte(mm, pmd) ((mm)->pmd_huge_pte)
Kirill A. Shutemov9a86cb72013-11-14 14:30:51 -08001599
Kirill A. Shutemove009bb32013-11-14 14:31:07 -08001600#endif
1601
Kirill A. Shutemov9a86cb72013-11-14 14:30:51 -08001602static inline spinlock_t *pmd_lock(struct mm_struct *mm, pmd_t *pmd)
1603{
1604 spinlock_t *ptl = pmd_lockptr(mm, pmd);
1605 spin_lock(ptl);
1606 return ptl;
1607}
1608
Linus Torvalds1da177e2005-04-16 15:20:36 -07001609extern void free_area_init(unsigned long * zones_size);
Johannes Weiner9109fb72008-07-23 21:27:20 -07001610extern void free_area_init_node(int nid, unsigned long * zones_size,
1611 unsigned long zone_start_pfn, unsigned long *zholes_size);
David Howells49a7f042012-03-28 18:30:03 +01001612extern void free_initmem(void);
1613
Jiang Liu69afade2013-04-29 15:06:21 -07001614/*
1615 * Free reserved pages within range [PAGE_ALIGN(start), end & PAGE_MASK)
1616 * into the buddy system. The freed pages will be poisoned with pattern
Jiang Liudbe67df2013-07-03 15:02:51 -07001617 * "poison" if it's within range [0, UCHAR_MAX].
Jiang Liu69afade2013-04-29 15:06:21 -07001618 * Return pages freed into the buddy system.
1619 */
Jiang Liu11199692013-07-03 15:02:48 -07001620extern unsigned long free_reserved_area(void *start, void *end,
Jiang Liu69afade2013-04-29 15:06:21 -07001621 int poison, char *s);
Jiang Liuc3d5f5f2013-07-03 15:03:14 -07001622
Jiang Liucfa11e02013-04-29 15:07:00 -07001623#ifdef CONFIG_HIGHMEM
1624/*
1625 * Free a highmem page into the buddy system, adjusting totalhigh_pages
1626 * and totalram_pages.
1627 */
1628extern void free_highmem_page(struct page *page);
1629#endif
Jiang Liu69afade2013-04-29 15:06:21 -07001630
Jiang Liuc3d5f5f2013-07-03 15:03:14 -07001631extern void adjust_managed_page_count(struct page *page, long count);
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07001632extern void mem_init_print_info(const char *str);
Jiang Liu69afade2013-04-29 15:06:21 -07001633
1634/* Free the reserved page into the buddy system, so it gets managed. */
1635static inline void __free_reserved_page(struct page *page)
1636{
1637 ClearPageReserved(page);
1638 init_page_count(page);
1639 __free_page(page);
1640}
1641
1642static inline void free_reserved_page(struct page *page)
1643{
1644 __free_reserved_page(page);
1645 adjust_managed_page_count(page, 1);
1646}
1647
1648static inline void mark_page_reserved(struct page *page)
1649{
1650 SetPageReserved(page);
1651 adjust_managed_page_count(page, -1);
1652}
1653
1654/*
1655 * Default method to free all the __init memory into the buddy system.
Jiang Liudbe67df2013-07-03 15:02:51 -07001656 * The freed pages will be poisoned with pattern "poison" if it's within
1657 * range [0, UCHAR_MAX].
1658 * Return pages freed into the buddy system.
Jiang Liu69afade2013-04-29 15:06:21 -07001659 */
1660static inline unsigned long free_initmem_default(int poison)
1661{
1662 extern char __init_begin[], __init_end[];
1663
Jiang Liu11199692013-07-03 15:02:48 -07001664 return free_reserved_area(&__init_begin, &__init_end,
Jiang Liu69afade2013-04-29 15:06:21 -07001665 poison, "unused kernel");
1666}
1667
Jiang Liu7ee3d4e2013-07-03 15:03:41 -07001668static inline unsigned long get_num_physpages(void)
1669{
1670 int nid;
1671 unsigned long phys_pages = 0;
1672
1673 for_each_online_node(nid)
1674 phys_pages += node_present_pages(nid);
1675
1676 return phys_pages;
1677}
1678
Tejun Heo0ee332c2011-12-08 10:22:09 -08001679#ifdef CONFIG_HAVE_MEMBLOCK_NODE_MAP
Mel Gormanc7132162006-09-27 01:49:43 -07001680/*
Tejun Heo0ee332c2011-12-08 10:22:09 -08001681 * With CONFIG_HAVE_MEMBLOCK_NODE_MAP set, an architecture may initialise its
Mel Gormanc7132162006-09-27 01:49:43 -07001682 * zones, allocate the backing mem_map and account for memory holes in a more
1683 * architecture independent manner. This is a substitute for creating the
1684 * zone_sizes[] and zholes_size[] arrays and passing them to
1685 * free_area_init_node()
1686 *
1687 * An architecture is expected to register range of page frames backed by
Tejun Heo0ee332c2011-12-08 10:22:09 -08001688 * physical memory with memblock_add[_node]() before calling
Mel Gormanc7132162006-09-27 01:49:43 -07001689 * free_area_init_nodes() passing in the PFN each zone ends at. At a basic
1690 * usage, an architecture is expected to do something like
1691 *
1692 * unsigned long max_zone_pfns[MAX_NR_ZONES] = {max_dma, max_normal_pfn,
1693 * max_highmem_pfn};
1694 * for_each_valid_physical_page_range()
Tejun Heo0ee332c2011-12-08 10:22:09 -08001695 * memblock_add_node(base, size, nid)
Mel Gormanc7132162006-09-27 01:49:43 -07001696 * free_area_init_nodes(max_zone_pfns);
1697 *
Tejun Heo0ee332c2011-12-08 10:22:09 -08001698 * free_bootmem_with_active_regions() calls free_bootmem_node() for each
1699 * registered physical page range. Similarly
1700 * sparse_memory_present_with_active_regions() calls memory_present() for
1701 * each range when SPARSEMEM is enabled.
Mel Gormanc7132162006-09-27 01:49:43 -07001702 *
1703 * See mm/page_alloc.c for more information on each function exposed by
Tejun Heo0ee332c2011-12-08 10:22:09 -08001704 * CONFIG_HAVE_MEMBLOCK_NODE_MAP.
Mel Gormanc7132162006-09-27 01:49:43 -07001705 */
1706extern void free_area_init_nodes(unsigned long *max_zone_pfn);
Tejun Heo1e019792011-07-12 09:45:34 +02001707unsigned long node_map_pfn_alignment(void);
Yinghai Lu32996252009-12-15 17:59:02 -08001708unsigned long __absent_pages_in_range(int nid, unsigned long start_pfn,
1709 unsigned long end_pfn);
Mel Gormanc7132162006-09-27 01:49:43 -07001710extern unsigned long absent_pages_in_range(unsigned long start_pfn,
1711 unsigned long end_pfn);
1712extern void get_pfn_range_for_nid(unsigned int nid,
1713 unsigned long *start_pfn, unsigned long *end_pfn);
1714extern unsigned long find_min_pfn_with_active_regions(void);
Mel Gormanc7132162006-09-27 01:49:43 -07001715extern void free_bootmem_with_active_regions(int nid,
1716 unsigned long max_low_pfn);
1717extern void sparse_memory_present_with_active_regions(int nid);
KAMEZAWA Hiroyukif2dbcfa2009-02-18 14:48:32 -08001718
Tejun Heo0ee332c2011-12-08 10:22:09 -08001719#endif /* CONFIG_HAVE_MEMBLOCK_NODE_MAP */
Tejun Heo5dfe8662011-07-14 09:46:10 +02001720
Tejun Heo0ee332c2011-12-08 10:22:09 -08001721#if !defined(CONFIG_HAVE_MEMBLOCK_NODE_MAP) && \
KAMEZAWA Hiroyukif2dbcfa2009-02-18 14:48:32 -08001722 !defined(CONFIG_HAVE_ARCH_EARLY_PFN_TO_NID)
1723static inline int __early_pfn_to_nid(unsigned long pfn)
1724{
1725 return 0;
1726}
1727#else
1728/* please see mm/page_alloc.c */
1729extern int __meminit early_pfn_to_nid(unsigned long pfn);
KAMEZAWA Hiroyukif2dbcfa2009-02-18 14:48:32 -08001730/* there is a per-arch backend function. */
1731extern int __meminit __early_pfn_to_nid(unsigned long pfn);
KAMEZAWA Hiroyukif2dbcfa2009-02-18 14:48:32 -08001732#endif
1733
Mel Gorman0e0b8642006-09-27 01:49:56 -07001734extern void set_dma_reserve(unsigned long new_dma_reserve);
Dave Hansena2f3aa022007-01-10 23:15:30 -08001735extern void memmap_init_zone(unsigned long, int, unsigned long,
1736 unsigned long, enum memmap_context);
Minchan Kimbc75d332009-06-16 15:32:48 -07001737extern void setup_per_zone_wmarks(void);
KOSAKI Motohiro1b79acc2011-05-24 17:11:32 -07001738extern int __meminit init_per_zone_wmark_min(void);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001739extern void mem_init(void);
David Howells8feae132009-01-08 12:04:47 +00001740extern void __init mmap_init(void);
David Rientjesb2b755b2011-03-24 15:18:15 -07001741extern void show_mem(unsigned int flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001742extern void si_meminfo(struct sysinfo * val);
1743extern void si_meminfo_node(struct sysinfo *val, int nid);
1744
Joe Perches3ee9a4f2011-10-31 17:08:35 -07001745extern __printf(3, 4)
1746void warn_alloc_failed(gfp_t gfp_mask, int order, const char *fmt, ...);
Dave Hansena238ab52011-05-24 17:12:16 -07001747
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07001748extern void setup_per_cpu_pageset(void);
Christoph Lametere7c8d5c2005-06-21 17:14:47 -07001749
Shaohua Li112067f2009-09-21 17:01:16 -07001750extern void zone_pcp_update(struct zone *zone);
Jiang Liu340175b2012-07-31 16:43:32 -07001751extern void zone_pcp_reset(struct zone *zone);
Shaohua Li112067f2009-09-21 17:01:16 -07001752
Paul Szabo75f7ad82013-02-22 16:34:42 -08001753/* page_alloc.c */
1754extern int min_free_kbytes;
1755
David Howells8feae132009-01-08 12:04:47 +00001756/* nommu.c */
David Howells33e5d7692009-04-02 16:56:32 -07001757extern atomic_long_t mmap_pages_allocated;
David Howells7e660872010-01-15 17:01:39 -08001758extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
David Howells8feae132009-01-08 12:04:47 +00001759
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -07001760/* interval_tree.c */
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -07001761void vma_interval_tree_insert(struct vm_area_struct *node,
1762 struct rb_root *root);
Michel Lespinasse9826a512012-10-08 16:31:35 -07001763void vma_interval_tree_insert_after(struct vm_area_struct *node,
1764 struct vm_area_struct *prev,
1765 struct rb_root *root);
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -07001766void vma_interval_tree_remove(struct vm_area_struct *node,
1767 struct rb_root *root);
1768struct vm_area_struct *vma_interval_tree_iter_first(struct rb_root *root,
1769 unsigned long start, unsigned long last);
1770struct vm_area_struct *vma_interval_tree_iter_next(struct vm_area_struct *node,
1771 unsigned long start, unsigned long last);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001772
Michel Lespinasse6b2dbba2012-10-08 16:31:25 -07001773#define vma_interval_tree_foreach(vma, root, start, last) \
1774 for (vma = vma_interval_tree_iter_first(root, start, last); \
1775 vma; vma = vma_interval_tree_iter_next(vma, start, last))
Linus Torvalds1da177e2005-04-16 15:20:36 -07001776
Michel Lespinassebf181b92012-10-08 16:31:39 -07001777void anon_vma_interval_tree_insert(struct anon_vma_chain *node,
1778 struct rb_root *root);
1779void anon_vma_interval_tree_remove(struct anon_vma_chain *node,
1780 struct rb_root *root);
1781struct anon_vma_chain *anon_vma_interval_tree_iter_first(
1782 struct rb_root *root, unsigned long start, unsigned long last);
1783struct anon_vma_chain *anon_vma_interval_tree_iter_next(
1784 struct anon_vma_chain *node, unsigned long start, unsigned long last);
Michel Lespinasseed8ea812012-10-08 16:31:45 -07001785#ifdef CONFIG_DEBUG_VM_RB
1786void anon_vma_interval_tree_verify(struct anon_vma_chain *node);
1787#endif
Michel Lespinassebf181b92012-10-08 16:31:39 -07001788
1789#define anon_vma_interval_tree_foreach(avc, root, start, last) \
1790 for (avc = anon_vma_interval_tree_iter_first(root, start, last); \
1791 avc; avc = anon_vma_interval_tree_iter_next(avc, start, last))
1792
Linus Torvalds1da177e2005-04-16 15:20:36 -07001793/* mmap.c */
Alan Cox34b4e4a2007-08-22 14:01:28 -07001794extern int __vm_enough_memory(struct mm_struct *mm, long pages, int cap_sys_admin);
Rik van Riel5beb4932010-03-05 13:42:07 -08001795extern int vma_adjust(struct vm_area_struct *vma, unsigned long start,
Linus Torvalds1da177e2005-04-16 15:20:36 -07001796 unsigned long end, pgoff_t pgoff, struct vm_area_struct *insert);
1797extern struct vm_area_struct *vma_merge(struct mm_struct *,
1798 struct vm_area_struct *prev, unsigned long addr, unsigned long end,
1799 unsigned long vm_flags, struct anon_vma *, struct file *, pgoff_t,
1800 struct mempolicy *);
1801extern struct anon_vma *find_mergeable_anon_vma(struct vm_area_struct *);
1802extern int split_vma(struct mm_struct *,
1803 struct vm_area_struct *, unsigned long addr, int new_below);
1804extern int insert_vm_struct(struct mm_struct *, struct vm_area_struct *);
1805extern void __vma_link_rb(struct mm_struct *, struct vm_area_struct *,
1806 struct rb_node **, struct rb_node *);
Hugh Dickinsa8fb5612005-10-29 18:15:57 -07001807extern void unlink_file_vma(struct vm_area_struct *);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001808extern struct vm_area_struct *copy_vma(struct vm_area_struct **,
Michel Lespinasse38a76012012-10-08 16:31:50 -07001809 unsigned long addr, unsigned long len, pgoff_t pgoff,
1810 bool *need_rmap_locks);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001811extern void exit_mmap(struct mm_struct *);
Matt Helsley925d1c42008-04-29 01:01:36 -07001812
Cyrill Gorcunov9c599022014-10-09 15:27:29 -07001813static inline int check_data_rlimit(unsigned long rlim,
1814 unsigned long new,
1815 unsigned long start,
1816 unsigned long end_data,
1817 unsigned long start_data)
1818{
1819 if (rlim < RLIM_INFINITY) {
1820 if (((new - start) + (end_data - start_data)) > rlim)
1821 return -ENOSPC;
1822 }
1823
1824 return 0;
1825}
1826
Andrea Arcangeli7906d002008-07-28 15:46:26 -07001827extern int mm_take_all_locks(struct mm_struct *mm);
1828extern void mm_drop_all_locks(struct mm_struct *mm);
1829
Jiri Slaby38646012011-05-26 16:25:46 -07001830extern void set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file);
1831extern struct file *get_mm_exe_file(struct mm_struct *mm);
Matt Helsley925d1c42008-04-29 01:01:36 -07001832
akpm@osdl.org119f6572005-05-01 08:58:35 -07001833extern int may_expand_vm(struct mm_struct *mm, unsigned long npages);
Stefani Seibold3935ed62014-03-17 23:22:02 +01001834extern struct vm_area_struct *_install_special_mapping(struct mm_struct *mm,
1835 unsigned long addr, unsigned long len,
Andy Lutomirskia62c34b2014-05-19 15:58:33 -07001836 unsigned long flags,
1837 const struct vm_special_mapping *spec);
1838/* This is an obsolete alternative to _install_special_mapping. */
Roland McGrathfa5dc222007-02-08 14:20:41 -08001839extern int install_special_mapping(struct mm_struct *mm,
1840 unsigned long addr, unsigned long len,
1841 unsigned long flags, struct page **pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001842
1843extern unsigned long get_unmapped_area(struct file *, unsigned long, unsigned long, unsigned long, unsigned long);
1844
Miklos Szeredi0165ab42007-07-15 23:38:26 -07001845extern unsigned long mmap_region(struct file *file, unsigned long addr,
Michel Lespinassec22c0d62013-02-22 16:32:43 -08001846 unsigned long len, vm_flags_t vm_flags, unsigned long pgoff);
Michel Lespinassebebeb3d2013-02-22 16:32:37 -08001847extern unsigned long do_mmap_pgoff(struct file *file, unsigned long addr,
1848 unsigned long len, unsigned long prot, unsigned long flags,
Michel Lespinasse41badc12013-02-22 16:32:47 -08001849 unsigned long pgoff, unsigned long *populate);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001850extern int do_munmap(struct mm_struct *, unsigned long, size_t);
1851
Michel Lespinassebebeb3d2013-02-22 16:32:37 -08001852#ifdef CONFIG_MMU
1853extern int __mm_populate(unsigned long addr, unsigned long len,
1854 int ignore_errors);
1855static inline void mm_populate(unsigned long addr, unsigned long len)
1856{
1857 /* Ignore errors */
1858 (void) __mm_populate(addr, len, 1);
1859}
1860#else
1861static inline void mm_populate(unsigned long addr, unsigned long len) {}
1862#endif
1863
Linus Torvaldse4eb1ff2012-04-20 15:35:40 -07001864/* These take the mm semaphore themselves */
1865extern unsigned long vm_brk(unsigned long, unsigned long);
Al Virobfce2812012-04-20 21:57:04 -04001866extern int vm_munmap(unsigned long, size_t);
Linus Torvalds6be5ceb2012-04-20 17:13:58 -07001867extern unsigned long vm_mmap(struct file *, unsigned long,
1868 unsigned long, unsigned long,
1869 unsigned long, unsigned long);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001870
Michel Lespinassedb4fbfb2012-12-11 16:01:49 -08001871struct vm_unmapped_area_info {
1872#define VM_UNMAPPED_AREA_TOPDOWN 1
1873 unsigned long flags;
1874 unsigned long length;
1875 unsigned long low_limit;
1876 unsigned long high_limit;
1877 unsigned long align_mask;
1878 unsigned long align_offset;
1879};
1880
1881extern unsigned long unmapped_area(struct vm_unmapped_area_info *info);
1882extern unsigned long unmapped_area_topdown(struct vm_unmapped_area_info *info);
1883
1884/*
1885 * Search for an unmapped address range.
1886 *
1887 * We are looking for a range that:
1888 * - does not intersect with any VMA;
1889 * - is contained within the [low_limit, high_limit) interval;
1890 * - is at least the desired size.
1891 * - satisfies (begin_addr & align_mask) == (align_offset & align_mask)
1892 */
1893static inline unsigned long
1894vm_unmapped_area(struct vm_unmapped_area_info *info)
1895{
Borislav Petkovcdd78752015-04-15 16:14:47 -07001896 if (info->flags & VM_UNMAPPED_AREA_TOPDOWN)
Michel Lespinassedb4fbfb2012-12-11 16:01:49 -08001897 return unmapped_area_topdown(info);
Borislav Petkovcdd78752015-04-15 16:14:47 -07001898 else
1899 return unmapped_area(info);
Michel Lespinassedb4fbfb2012-12-11 16:01:49 -08001900}
1901
Hugh Dickins85821aa2011-07-25 17:12:23 -07001902/* truncate.c */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001903extern void truncate_inode_pages(struct address_space *, loff_t);
Hans Reiserd7339072006-01-06 00:10:36 -08001904extern void truncate_inode_pages_range(struct address_space *,
1905 loff_t lstart, loff_t lend);
Johannes Weiner91b0abe2014-04-03 14:47:49 -07001906extern void truncate_inode_pages_final(struct address_space *);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001907
1908/* generic vm_area_ops exported for stackable file systems */
Nick Piggind0217ac2007-07-19 01:47:03 -07001909extern int filemap_fault(struct vm_area_struct *, struct vm_fault *);
Kirill A. Shutemovf1820362014-04-07 15:37:19 -07001910extern void filemap_map_pages(struct vm_area_struct *vma, struct vm_fault *vmf);
Jan Kara4fcf1c62012-06-12 16:20:29 +02001911extern int filemap_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001912
1913/* mm/page-writeback.c */
1914int write_one_page(struct page *page, int wait);
Nick Piggin1cf6e7d2009-02-18 14:48:18 -08001915void task_dirty_inc(struct task_struct *tsk);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001916
1917/* readahead.c */
1918#define VM_MAX_READAHEAD 128 /* kbytes */
1919#define VM_MIN_READAHEAD 16 /* kbytes (includes current page) */
Linus Torvalds1da177e2005-04-16 15:20:36 -07001920
Linus Torvalds1da177e2005-04-16 15:20:36 -07001921int force_page_cache_readahead(struct address_space *mapping, struct file *filp,
Andrew Morton7361f4d2005-11-07 00:59:28 -08001922 pgoff_t offset, unsigned long nr_to_read);
Rusty Russellcf914a72007-07-19 01:48:08 -07001923
1924void page_cache_sync_readahead(struct address_space *mapping,
1925 struct file_ra_state *ra,
1926 struct file *filp,
1927 pgoff_t offset,
1928 unsigned long size);
1929
1930void page_cache_async_readahead(struct address_space *mapping,
1931 struct file_ra_state *ra,
1932 struct file *filp,
1933 struct page *pg,
1934 pgoff_t offset,
1935 unsigned long size);
1936
Linus Torvalds1da177e2005-04-16 15:20:36 -07001937unsigned long max_sane_readahead(unsigned long nr);
1938
Michal Hockod05f3162011-05-24 17:11:44 -07001939/* Generic expand stack which grows the stack according to GROWS{UP,DOWN} */
Hugh Dickins46dea3d2005-10-29 18:16:20 -07001940extern int expand_stack(struct vm_area_struct *vma, unsigned long address);
Michal Hockod05f3162011-05-24 17:11:44 -07001941
1942/* CONFIG_STACK_GROWSUP still needs to to grow downwards at some places */
1943extern int expand_downwards(struct vm_area_struct *vma,
1944 unsigned long address);
Luck, Tony8ca3eb02010-08-24 11:44:18 -07001945#if VM_GROWSUP
Hugh Dickins46dea3d2005-10-29 18:16:20 -07001946extern int expand_upwards(struct vm_area_struct *vma, unsigned long address);
Luck, Tony8ca3eb02010-08-24 11:44:18 -07001947#else
Linus Torvaldsfee7e492015-01-06 13:00:05 -08001948 #define expand_upwards(vma, address) (0)
Matthew Wilcox9ab88512005-11-18 16:16:42 -05001949#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -07001950
1951/* Look up the first VMA which satisfies addr < vm_end, NULL if none. */
1952extern struct vm_area_struct * find_vma(struct mm_struct * mm, unsigned long addr);
1953extern struct vm_area_struct * find_vma_prev(struct mm_struct * mm, unsigned long addr,
1954 struct vm_area_struct **pprev);
1955
1956/* Look up the first VMA which intersects the interval start_addr..end_addr-1,
1957 NULL if none. Assume start_addr < end_addr. */
1958static inline struct vm_area_struct * find_vma_intersection(struct mm_struct * mm, unsigned long start_addr, unsigned long end_addr)
1959{
1960 struct vm_area_struct * vma = find_vma(mm,start_addr);
1961
1962 if (vma && end_addr <= vma->vm_start)
1963 vma = NULL;
1964 return vma;
1965}
1966
1967static inline unsigned long vma_pages(struct vm_area_struct *vma)
1968{
1969 return (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
1970}
1971
Pavel Emelyanov640708a2012-01-10 15:11:23 -08001972/* Look up the first VMA which exactly match the interval vm_start ... vm_end */
1973static inline struct vm_area_struct *find_exact_vma(struct mm_struct *mm,
1974 unsigned long vm_start, unsigned long vm_end)
1975{
1976 struct vm_area_struct *vma = find_vma(mm, vm_start);
1977
1978 if (vma && (vma->vm_start != vm_start || vma->vm_end != vm_end))
1979 vma = NULL;
1980
1981 return vma;
1982}
1983
David Howellsbad849b2010-08-26 16:00:34 +01001984#ifdef CONFIG_MMU
Hugh Dickins804af2c2006-07-26 21:39:49 +01001985pgprot_t vm_get_page_prot(unsigned long vm_flags);
Peter Feiner64e45502014-10-13 15:55:46 -07001986void vma_set_page_prot(struct vm_area_struct *vma);
David Howellsbad849b2010-08-26 16:00:34 +01001987#else
1988static inline pgprot_t vm_get_page_prot(unsigned long vm_flags)
1989{
1990 return __pgprot(0);
1991}
Peter Feiner64e45502014-10-13 15:55:46 -07001992static inline void vma_set_page_prot(struct vm_area_struct *vma)
1993{
1994 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
1995}
David Howellsbad849b2010-08-26 16:00:34 +01001996#endif
1997
Aneesh Kumar K.V58772312013-12-06 00:08:22 +05301998#ifdef CONFIG_NUMA_BALANCING
Mel Gorman4b10e7d2012-10-25 14:16:32 +02001999unsigned long change_prot_numa(struct vm_area_struct *vma,
Lee Schermerhornb24f53a2012-10-25 14:16:32 +02002000 unsigned long start, unsigned long end);
2001#endif
2002
Hugh Dickinsdeceb6c2005-10-29 18:16:33 -07002003struct vm_area_struct *find_extend_vma(struct mm_struct *, unsigned long addr);
Hugh Dickinsdeceb6c2005-10-29 18:16:33 -07002004int remap_pfn_range(struct vm_area_struct *, unsigned long addr,
2005 unsigned long pfn, unsigned long size, pgprot_t);
Linus Torvaldsa145dd42005-11-30 09:35:19 -08002006int vm_insert_page(struct vm_area_struct *, unsigned long addr, struct page *);
Nick Piggine0dc0d82007-02-12 00:51:36 -08002007int vm_insert_pfn(struct vm_area_struct *vma, unsigned long addr,
2008 unsigned long pfn);
Nick Piggin423bad62008-04-28 02:13:01 -07002009int vm_insert_mixed(struct vm_area_struct *vma, unsigned long addr,
2010 unsigned long pfn);
Linus Torvaldsb4cbb192013-04-16 13:45:37 -07002011int vm_iomap_memory(struct vm_area_struct *vma, phys_addr_t start, unsigned long len);
2012
Linus Torvalds1da177e2005-04-16 15:20:36 -07002013
Michel Lespinasse240aade2013-02-22 16:35:56 -08002014struct page *follow_page_mask(struct vm_area_struct *vma,
2015 unsigned long address, unsigned int foll_flags,
2016 unsigned int *page_mask);
2017
2018static inline struct page *follow_page(struct vm_area_struct *vma,
2019 unsigned long address, unsigned int foll_flags)
2020{
2021 unsigned int unused_page_mask;
2022 return follow_page_mask(vma, address, foll_flags, &unused_page_mask);
2023}
2024
Hugh Dickinsdeceb6c2005-10-29 18:16:33 -07002025#define FOLL_WRITE 0x01 /* check pte is writable */
2026#define FOLL_TOUCH 0x02 /* mark page accessed */
2027#define FOLL_GET 0x04 /* do get_page on page */
Hugh Dickins8e4b9a602009-09-21 17:03:26 -07002028#define FOLL_DUMP 0x08 /* give error on hole if it would be zero */
Hugh Dickins58fa8792009-09-21 17:03:31 -07002029#define FOLL_FORCE 0x10 /* get_user_pages read/write w/o permission */
Gleb Natapov318b2752011-03-22 16:30:51 -07002030#define FOLL_NOWAIT 0x20 /* if a disk transfer is needed, start the IO
2031 * and return without waiting upon it */
Kirill A. Shutemov84d33df2015-04-14 15:44:37 -07002032#define FOLL_POPULATE 0x40 /* fault in page */
Andrea Arcangeli500d65d2011-01-13 15:46:55 -08002033#define FOLL_SPLIT 0x80 /* don't return transhuge pages, split them */
Huang Ying69ebb832011-01-30 11:15:48 +08002034#define FOLL_HWPOISON 0x100 /* check page is hwpoisoned */
Andrea Arcangeli0b9d7052012-10-05 21:36:27 +02002035#define FOLL_NUMA 0x200 /* force NUMA hinting page fault */
Hugh Dickins5117b3b2013-02-22 16:36:07 -08002036#define FOLL_MIGRATION 0x400 /* wait for page to replace migration entry */
Andres Lagar-Cavilla234b2392014-09-17 10:51:48 -07002037#define FOLL_TRIED 0x800 /* a retry, previous pass started an IO */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002038
Martin Schwidefsky2f569af2008-02-08 04:22:04 -08002039typedef int (*pte_fn_t)(pte_t *pte, pgtable_t token, unsigned long addr,
Jeremy Fitzhardingeaee16b32007-05-06 14:48:54 -07002040 void *data);
2041extern int apply_to_page_range(struct mm_struct *mm, unsigned long address,
2042 unsigned long size, pte_fn_t fn, void *data);
2043
Linus Torvalds1da177e2005-04-16 15:20:36 -07002044#ifdef CONFIG_PROC_FS
Hugh Dickinsab50b8e2005-10-29 18:15:56 -07002045void vm_stat_account(struct mm_struct *, unsigned long, struct file *, long);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002046#else
Hugh Dickinsab50b8e2005-10-29 18:15:56 -07002047static inline void vm_stat_account(struct mm_struct *mm,
Linus Torvalds1da177e2005-04-16 15:20:36 -07002048 unsigned long flags, struct file *file, long pages)
2049{
Huang Shijie44de9d02012-07-31 16:41:49 -07002050 mm->total_vm += pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -07002051}
2052#endif /* CONFIG_PROC_FS */
2053
Ingo Molnar12d6f212008-01-30 13:33:58 +01002054#ifdef CONFIG_DEBUG_PAGEALLOC
Joonsoo Kim031bc572014-12-12 16:55:52 -08002055extern bool _debug_pagealloc_enabled;
2056extern void __kernel_map_pages(struct page *page, int numpages, int enable);
2057
2058static inline bool debug_pagealloc_enabled(void)
2059{
2060 return _debug_pagealloc_enabled;
2061}
2062
2063static inline void
2064kernel_map_pages(struct page *page, int numpages, int enable)
2065{
2066 if (!debug_pagealloc_enabled())
2067 return;
2068
2069 __kernel_map_pages(page, numpages, enable);
2070}
Rafael J. Wysocki8a235ef2008-02-20 01:47:44 +01002071#ifdef CONFIG_HIBERNATION
2072extern bool kernel_page_present(struct page *page);
2073#endif /* CONFIG_HIBERNATION */
Ingo Molnar12d6f212008-01-30 13:33:58 +01002074#else
Linus Torvalds1da177e2005-04-16 15:20:36 -07002075static inline void
Nick Piggin9858db52006-10-11 01:21:30 -07002076kernel_map_pages(struct page *page, int numpages, int enable) {}
Rafael J. Wysocki8a235ef2008-02-20 01:47:44 +01002077#ifdef CONFIG_HIBERNATION
2078static inline bool kernel_page_present(struct page *page) { return true; }
2079#endif /* CONFIG_HIBERNATION */
Linus Torvalds1da177e2005-04-16 15:20:36 -07002080#endif
2081
Andy Lutomirskia6c19df2014-08-08 14:23:40 -07002082#ifdef __HAVE_ARCH_GATE_AREA
Stephen Wilson31db58b2011-03-13 15:49:15 -04002083extern struct vm_area_struct *get_gate_vma(struct mm_struct *mm);
Andy Lutomirskia6c19df2014-08-08 14:23:40 -07002084extern int in_gate_area_no_mm(unsigned long addr);
2085extern int in_gate_area(struct mm_struct *mm, unsigned long addr);
Linus Torvalds1da177e2005-04-16 15:20:36 -07002086#else
Andy Lutomirskia6c19df2014-08-08 14:23:40 -07002087static inline struct vm_area_struct *get_gate_vma(struct mm_struct *mm)
2088{
2089 return NULL;
2090}
2091static inline int in_gate_area_no_mm(unsigned long addr) { return 0; }
2092static inline int in_gate_area(struct mm_struct *mm, unsigned long addr)
2093{
2094 return 0;
2095}
Linus Torvalds1da177e2005-04-16 15:20:36 -07002096#endif /* __HAVE_ARCH_GATE_AREA */
2097
Josh Triplett146732c2013-04-29 15:07:22 -07002098#ifdef CONFIG_SYSCTL
2099extern int sysctl_drop_caches;
Alexey Dobriyan8d65af72009-09-23 15:57:19 -07002100int drop_caches_sysctl_handler(struct ctl_table *, int,
Andrew Morton9d0243b2006-01-08 01:00:39 -08002101 void __user *, size_t *, loff_t *);
Josh Triplett146732c2013-04-29 15:07:22 -07002102#endif
2103
Vladimir Davydovcb731d62015-02-12 14:58:54 -08002104void drop_slab(void);
2105void drop_slab_node(int nid);
Andrew Morton9d0243b2006-01-08 01:00:39 -08002106
Luke Yang7a9166e2006-02-20 18:28:07 -08002107#ifndef CONFIG_MMU
2108#define randomize_va_space 0
2109#else
Andi Kleena62eaf12006-02-16 23:41:58 +01002110extern int randomize_va_space;
Luke Yang7a9166e2006-02-20 18:28:07 -08002111#endif
Andi Kleena62eaf12006-02-16 23:41:58 +01002112
Sam Ravnborg045e72a2007-07-26 10:41:13 -07002113const char * arch_vma_name(struct vm_area_struct *vma);
Andi Kleen03252912008-01-30 13:33:18 +01002114void print_vma_addr(char *prefix, unsigned long rip);
Ingo Molnare6e54942006-06-27 02:53:50 -07002115
Yinghai Lu9bdac912010-02-10 01:20:22 -08002116void sparse_mem_maps_populate_node(struct page **map_map,
2117 unsigned long pnum_begin,
2118 unsigned long pnum_end,
2119 unsigned long map_count,
2120 int nodeid);
2121
Yasunori Goto98f3cfc2007-10-16 01:26:14 -07002122struct page *sparse_mem_map_populate(unsigned long pnum, int nid);
Andy Whitcroft29c71112007-10-16 01:24:14 -07002123pgd_t *vmemmap_pgd_populate(unsigned long addr, int node);
2124pud_t *vmemmap_pud_populate(pgd_t *pgd, unsigned long addr, int node);
2125pmd_t *vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node);
2126pte_t *vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node);
Christoph Lameter8f6aac42007-10-16 01:24:13 -07002127void *vmemmap_alloc_block(unsigned long size, int node);
Yinghai Lu9bdac912010-02-10 01:20:22 -08002128void *vmemmap_alloc_block_buf(unsigned long size, int node);
Christoph Lameter8f6aac42007-10-16 01:24:13 -07002129void vmemmap_verify(pte_t *, int, unsigned long, unsigned long);
Johannes Weiner0aad8182013-04-29 15:07:50 -07002130int vmemmap_populate_basepages(unsigned long start, unsigned long end,
2131 int node);
2132int vmemmap_populate(unsigned long start, unsigned long end, int node);
Yinghai Luc2b91e22008-04-12 01:19:24 -07002133void vmemmap_populate_print_last(void);
Tang Chen01975182013-02-22 16:33:08 -08002134#ifdef CONFIG_MEMORY_HOTPLUG
Johannes Weiner0aad8182013-04-29 15:07:50 -07002135void vmemmap_free(unsigned long start, unsigned long end);
Tang Chen01975182013-02-22 16:33:08 -08002136#endif
Yasuaki Ishimatsu46723bf2013-02-22 16:33:00 -08002137void register_page_bootmem_memmap(unsigned long section_nr, struct page *map,
2138 unsigned long size);
Andi Kleen6a460792009-09-16 11:50:15 +02002139
Andi Kleen82ba0112009-12-16 12:19:57 +01002140enum mf_flags {
2141 MF_COUNT_INCREASED = 1 << 0,
Tony Luck7329bbe2011-12-13 09:27:58 -08002142 MF_ACTION_REQUIRED = 1 << 1,
Tony Luck6751ed62012-07-11 10:20:47 -07002143 MF_MUST_KILL = 1 << 2,
Naveen N. Raocf870c72013-07-10 14:57:01 +05302144 MF_SOFT_OFFLINE = 1 << 3,
Andi Kleen82ba0112009-12-16 12:19:57 +01002145};
Tony Luckcd42f4a2011-12-15 10:48:12 -08002146extern int memory_failure(unsigned long pfn, int trapno, int flags);
Huang Yingea8f5fb2011-07-13 13:14:27 +08002147extern void memory_failure_queue(unsigned long pfn, int trapno, int flags);
Wu Fengguang847ce402009-12-16 12:19:58 +01002148extern int unpoison_memory(unsigned long pfn);
Andi Kleen6a460792009-09-16 11:50:15 +02002149extern int sysctl_memory_failure_early_kill;
2150extern int sysctl_memory_failure_recovery;
Andi Kleenfacb6012009-12-16 12:20:00 +01002151extern void shake_page(struct page *p, int access);
Xishi Qiu293c07e2013-02-22 16:34:02 -08002152extern atomic_long_t num_poisoned_pages;
Andi Kleenfacb6012009-12-16 12:20:00 +01002153extern int soft_offline_page(struct page *page, int flags);
Andi Kleen6a460792009-09-16 11:50:15 +02002154
Andrea Arcangeli47ad8472011-01-13 15:46:47 -08002155#if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLBFS)
2156extern void clear_huge_page(struct page *page,
2157 unsigned long addr,
2158 unsigned int pages_per_huge_page);
2159extern void copy_user_huge_page(struct page *dst, struct page *src,
2160 unsigned long addr, struct vm_area_struct *vma,
2161 unsigned int pages_per_huge_page);
2162#endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLBFS */
2163
Joonsoo Kime30825f2014-12-12 16:55:49 -08002164extern struct page_ext_operations debug_guardpage_ops;
2165extern struct page_ext_operations page_poisoning_ops;
2166
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -08002167#ifdef CONFIG_DEBUG_PAGEALLOC
2168extern unsigned int _debug_guardpage_minorder;
Joonsoo Kime30825f2014-12-12 16:55:49 -08002169extern bool _debug_guardpage_enabled;
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -08002170
2171static inline unsigned int debug_guardpage_minorder(void)
2172{
2173 return _debug_guardpage_minorder;
2174}
2175
Joonsoo Kime30825f2014-12-12 16:55:49 -08002176static inline bool debug_guardpage_enabled(void)
2177{
2178 return _debug_guardpage_enabled;
2179}
2180
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -08002181static inline bool page_is_guard(struct page *page)
2182{
Joonsoo Kime30825f2014-12-12 16:55:49 -08002183 struct page_ext *page_ext;
2184
2185 if (!debug_guardpage_enabled())
2186 return false;
2187
2188 page_ext = lookup_page_ext(page);
2189 return test_bit(PAGE_EXT_DEBUG_GUARD, &page_ext->flags);
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -08002190}
2191#else
2192static inline unsigned int debug_guardpage_minorder(void) { return 0; }
Joonsoo Kime30825f2014-12-12 16:55:49 -08002193static inline bool debug_guardpage_enabled(void) { return false; }
Stanislaw Gruszkac0a32fc52012-01-10 15:07:28 -08002194static inline bool page_is_guard(struct page *page) { return false; }
2195#endif /* CONFIG_DEBUG_PAGEALLOC */
2196
Cody P Schaferf9872ca2013-04-29 15:08:01 -07002197#if MAX_NUMNODES > 1
2198void __init setup_nr_node_ids(void);
2199#else
2200static inline void setup_nr_node_ids(void) {}
2201#endif
2202
Linus Torvalds1da177e2005-04-16 15:20:36 -07002203#endif /* __KERNEL__ */
2204#endif /* _LINUX_MM_H */