blob: 830e65dc01ee1f264e715632f2f99fd95c7bd529 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef _LINUX_RMAP_H
2#define _LINUX_RMAP_H
3/*
4 * Declarations for Reverse Mapping functions in mm/rmap.c
5 */
6
Linus Torvalds1da177e2005-04-16 15:20:36 -07007#include <linux/list.h>
8#include <linux/slab.h>
9#include <linux/mm.h>
10#include <linux/spinlock.h>
Balbir Singhbed71612008-02-07 00:14:01 -080011#include <linux/memcontrol.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012
13/*
14 * The anon_vma heads a list of private "related" vmas, to scan if
15 * an anonymous page pointing to this anon_vma needs to be unmapped:
16 * the vmas on the list will be related by forking, or by splitting.
17 *
18 * Since vmas come and go as they are split and merged (particularly
19 * in mprotect), the mapping field of an anonymous page cannot point
20 * directly to a vma: instead it points to an anon_vma, on whose list
21 * the related vmas can be easily linked or unlinked.
22 *
23 * After unlinking the last vma on the list, we must garbage collect
24 * the anon_vma object itself: we're guaranteed no page can be
25 * pointing to this anon_vma once its vma list is empty.
26 */
27struct anon_vma {
Rik van Riel5c341ee2010-08-09 17:18:39 -070028 struct anon_vma *root; /* Root of this anon_vma tree */
Richard Kennedy182fea82010-10-26 14:21:55 -070029 spinlock_t lock; /* Serialize access to vma list */
Mel Gorman7f60c212010-05-24 14:32:18 -070030 /*
Peter Zijlstra83813262011-03-22 16:32:48 -070031 * The refcount is taken on an anon_vma when there is no
Mel Gorman7f60c212010-05-24 14:32:18 -070032 * guarantee that the vma of page tables will exist for
33 * the duration of the operation. A caller that takes
34 * the reference is responsible for clearing up the
35 * anon_vma if they are the last user on release
36 */
Peter Zijlstra83813262011-03-22 16:32:48 -070037 atomic_t refcount;
38
Andrea Arcangeli7906d002008-07-28 15:46:26 -070039 /*
40 * NOTE: the LSB of the head.next is set by
41 * mm_take_all_locks() _after_ taking the above lock. So the
42 * head must only be read/written after taking the above lock
43 * to be sure to see a valid next pointer. The LSB bit itself
44 * is serialized by a system wide lock only visible to
45 * mm_take_all_locks() (mm_all_locks_mutex).
46 */
Rik van Riel5beb4932010-03-05 13:42:07 -080047 struct list_head head; /* Chain of private "related" vmas */
48};
49
50/*
51 * The copy-on-write semantics of fork mean that an anon_vma
52 * can become associated with multiple processes. Furthermore,
53 * each child process will have its own anon_vma, where new
54 * pages for that process are instantiated.
55 *
56 * This structure allows us to find the anon_vmas associated
57 * with a VMA, or the VMAs associated with an anon_vma.
58 * The "same_vma" list contains the anon_vma_chains linking
59 * all the anon_vmas associated with this VMA.
60 * The "same_anon_vma" list contains the anon_vma_chains
61 * which link all the VMAs associated with this anon_vma.
62 */
63struct anon_vma_chain {
64 struct vm_area_struct *vma;
65 struct anon_vma *anon_vma;
66 struct list_head same_vma; /* locked by mmap_sem & page_table_lock */
67 struct list_head same_anon_vma; /* locked by anon_vma->lock */
Linus Torvalds1da177e2005-04-16 15:20:36 -070068};
69
70#ifdef CONFIG_MMU
Rik van Riel76545062010-08-09 17:18:41 -070071static inline void get_anon_vma(struct anon_vma *anon_vma)
72{
Peter Zijlstra83813262011-03-22 16:32:48 -070073 atomic_inc(&anon_vma->refcount);
Rik van Riel76545062010-08-09 17:18:41 -070074}
75
Peter Zijlstra01d8b202011-03-22 16:32:49 -070076void __put_anon_vma(struct anon_vma *anon_vma);
77
78static inline void put_anon_vma(struct anon_vma *anon_vma)
79{
80 if (atomic_dec_and_test(&anon_vma->refcount))
81 __put_anon_vma(anon_vma);
82}
Linus Torvalds1da177e2005-04-16 15:20:36 -070083
Hugh Dickins3ca7b3c2009-12-14 17:58:57 -080084static inline struct anon_vma *page_anon_vma(struct page *page)
85{
86 if (((unsigned long)page->mapping & PAGE_MAPPING_FLAGS) !=
87 PAGE_MAPPING_ANON)
88 return NULL;
89 return page_rmapping(page);
90}
91
Rik van Rielbb4a3402010-08-09 17:18:37 -070092static inline void vma_lock_anon_vma(struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -070093{
94 struct anon_vma *anon_vma = vma->anon_vma;
95 if (anon_vma)
Rik van Riel012f1802010-08-09 17:18:40 -070096 spin_lock(&anon_vma->root->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -070097}
98
Rik van Rielbb4a3402010-08-09 17:18:37 -070099static inline void vma_unlock_anon_vma(struct vm_area_struct *vma)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100{
101 struct anon_vma *anon_vma = vma->anon_vma;
102 if (anon_vma)
Rik van Riel012f1802010-08-09 17:18:40 -0700103 spin_unlock(&anon_vma->root->lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104}
105
Rik van Rielcba48b92010-08-09 17:18:38 -0700106static inline void anon_vma_lock(struct anon_vma *anon_vma)
107{
Rik van Riel012f1802010-08-09 17:18:40 -0700108 spin_lock(&anon_vma->root->lock);
Rik van Rielcba48b92010-08-09 17:18:38 -0700109}
110
111static inline void anon_vma_unlock(struct anon_vma *anon_vma)
112{
Rik van Riel012f1802010-08-09 17:18:40 -0700113 spin_unlock(&anon_vma->root->lock);
Rik van Rielcba48b92010-08-09 17:18:38 -0700114}
115
Linus Torvalds1da177e2005-04-16 15:20:36 -0700116/*
117 * anon_vma helper functions.
118 */
119void anon_vma_init(void); /* create anon_vma_cachep */
120int anon_vma_prepare(struct vm_area_struct *);
Rik van Riel5beb4932010-03-05 13:42:07 -0800121void unlink_anon_vmas(struct vm_area_struct *);
122int anon_vma_clone(struct vm_area_struct *, struct vm_area_struct *);
123int anon_vma_fork(struct vm_area_struct *, struct vm_area_struct *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700124void __anon_vma_link(struct vm_area_struct *);
125
Rik van Riel5beb4932010-03-05 13:42:07 -0800126static inline void anon_vma_merge(struct vm_area_struct *vma,
127 struct vm_area_struct *next)
128{
129 VM_BUG_ON(vma->anon_vma != next->anon_vma);
130 unlink_anon_vmas(next);
131}
132
Peter Zijlstra01d8b202011-03-22 16:32:49 -0700133struct anon_vma *page_get_anon_vma(struct page *page);
134
Linus Torvalds1da177e2005-04-16 15:20:36 -0700135/*
136 * rmap interfaces called when adding or removing pte of page
137 */
Rik van Rielc44b6742010-03-05 13:42:09 -0800138void page_move_anon_rmap(struct page *, struct vm_area_struct *, unsigned long);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139void page_add_anon_rmap(struct page *, struct vm_area_struct *, unsigned long);
Rik van Rielad8c2ee2010-08-09 17:19:48 -0700140void do_page_add_anon_rmap(struct page *, struct vm_area_struct *,
141 unsigned long, int);
Nick Piggin9617d952006-01-06 00:11:12 -0800142void page_add_new_anon_rmap(struct page *, struct vm_area_struct *, unsigned long);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700143void page_add_file_rmap(struct page *);
Hugh Dickinsedc315f2009-01-06 14:40:11 -0800144void page_remove_rmap(struct page *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700145
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +0900146void hugepage_add_anon_rmap(struct page *, struct vm_area_struct *,
147 unsigned long);
148void hugepage_add_new_anon_rmap(struct page *, struct vm_area_struct *,
149 unsigned long);
150
Hugh Dickins21333b22009-09-21 17:01:59 -0700151static inline void page_dup_rmap(struct page *page)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700152{
153 atomic_inc(&page->_mapcount);
154}
155
156/*
157 * Called from mm/vmscan.c to handle paging out
158 */
Wu Fengguang6fe6b7e2009-06-16 15:33:05 -0700159int page_referenced(struct page *, int is_locked,
160 struct mem_cgroup *cnt, unsigned long *vm_flags);
Hugh Dickins5ad64682009-12-14 17:59:24 -0800161int page_referenced_one(struct page *, struct vm_area_struct *,
162 unsigned long address, unsigned int *mapcount, unsigned long *vm_flags);
163
Andi Kleen14fa31b2009-09-16 11:50:10 +0200164enum ttu_flags {
165 TTU_UNMAP = 0, /* unmap mode */
166 TTU_MIGRATION = 1, /* migration mode */
167 TTU_MUNLOCK = 2, /* munlock mode */
168 TTU_ACTION_MASK = 0xff,
169
170 TTU_IGNORE_MLOCK = (1 << 8), /* ignore mlock */
171 TTU_IGNORE_ACCESS = (1 << 9), /* don't age */
Andi Kleen888b9f72009-09-16 11:50:11 +0200172 TTU_IGNORE_HWPOISON = (1 << 10),/* corrupted page is recoverable */
Andi Kleen14fa31b2009-09-16 11:50:10 +0200173};
174#define TTU_ACTION(x) ((x) & TTU_ACTION_MASK)
175
Andrea Arcangeli71e3aac2011-01-13 15:46:52 -0800176bool is_vma_temporary_stack(struct vm_area_struct *vma);
177
Andi Kleen14fa31b2009-09-16 11:50:10 +0200178int try_to_unmap(struct page *, enum ttu_flags flags);
Hugh Dickins5ad64682009-12-14 17:59:24 -0800179int try_to_unmap_one(struct page *, struct vm_area_struct *,
180 unsigned long address, enum ttu_flags flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700181
182/*
Carsten Otteceffc072005-06-23 22:05:25 -0700183 * Called from mm/filemap_xip.c to unmap empty zero page
184 */
Namhyung Kime9a81a82010-10-26 14:22:01 -0700185pte_t *__page_check_address(struct page *, struct mm_struct *,
Nick Piggin479db0b2008-08-20 14:09:18 -0700186 unsigned long, spinlock_t **, int);
Carsten Otteceffc072005-06-23 22:05:25 -0700187
Namhyung Kime9a81a82010-10-26 14:22:01 -0700188static inline pte_t *page_check_address(struct page *page, struct mm_struct *mm,
189 unsigned long address,
190 spinlock_t **ptlp, int sync)
191{
192 pte_t *ptep;
193
194 __cond_lock(*ptlp, ptep = __page_check_address(page, mm, address,
195 ptlp, sync));
196 return ptep;
197}
198
Carsten Otteceffc072005-06-23 22:05:25 -0700199/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700200 * Used by swapoff to help locate where page is expected in vma.
201 */
202unsigned long page_address_in_vma(struct page *, struct vm_area_struct *);
203
Peter Zijlstrad08b3852006-09-25 23:30:57 -0700204/*
205 * Cleans the PTEs of shared mappings.
206 * (and since clean PTEs should also be readonly, write protects them too)
207 *
208 * returns the number of cleaned PTEs.
209 */
210int page_mkclean(struct page *);
211
Nick Pigginb291f002008-10-18 20:26:44 -0700212/*
213 * called in munlock()/munmap() path to check for other vmas holding
214 * the page mlocked.
215 */
216int try_to_munlock(struct page *);
Nick Pigginb291f002008-10-18 20:26:44 -0700217
Andi Kleen10be22d2009-09-16 11:50:04 +0200218/*
219 * Called by memory-failure.c to kill processes.
220 */
Namhyung Kimea4525b2010-10-26 14:22:01 -0700221struct anon_vma *__page_lock_anon_vma(struct page *page);
222
223static inline struct anon_vma *page_lock_anon_vma(struct page *page)
224{
225 struct anon_vma *anon_vma;
226
227 __cond_lock(RCU, anon_vma = __page_lock_anon_vma(page));
228
229 /* (void) is needed to make gcc happy */
230 (void) __cond_lock(&anon_vma->root->lock, anon_vma);
231
232 return anon_vma;
233}
234
Andi Kleen10be22d2009-09-16 11:50:04 +0200235void page_unlock_anon_vma(struct anon_vma *anon_vma);
Andi Kleen6a460792009-09-16 11:50:15 +0200236int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma);
Andi Kleen10be22d2009-09-16 11:50:04 +0200237
Hugh Dickinse9995ef2009-12-14 17:59:31 -0800238/*
239 * Called by migrate.c to remove migration ptes, but might be used more later.
240 */
241int rmap_walk(struct page *page, int (*rmap_one)(struct page *,
242 struct vm_area_struct *, unsigned long, void *), void *arg);
243
Linus Torvalds1da177e2005-04-16 15:20:36 -0700244#else /* !CONFIG_MMU */
245
246#define anon_vma_init() do {} while (0)
247#define anon_vma_prepare(vma) (0)
248#define anon_vma_link(vma) do {} while (0)
249
Mike Frysinger01ff53f2009-06-23 12:37:01 -0700250static inline int page_referenced(struct page *page, int is_locked,
251 struct mem_cgroup *cnt,
252 unsigned long *vm_flags)
253{
254 *vm_flags = 0;
Johannes Weiner64574742010-03-05 13:42:22 -0800255 return 0;
Mike Frysinger01ff53f2009-06-23 12:37:01 -0700256}
257
Christoph Lametera48d07a2006-02-01 03:05:38 -0800258#define try_to_unmap(page, refs) SWAP_FAIL
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259
Peter Zijlstrad08b3852006-09-25 23:30:57 -0700260static inline int page_mkclean(struct page *page)
261{
262 return 0;
263}
264
265
Linus Torvalds1da177e2005-04-16 15:20:36 -0700266#endif /* CONFIG_MMU */
267
268/*
269 * Return values of try_to_unmap
270 */
271#define SWAP_SUCCESS 0
272#define SWAP_AGAIN 1
273#define SWAP_FAIL 2
Nick Pigginb291f002008-10-18 20:26:44 -0700274#define SWAP_MLOCK 3
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275
276#endif /* _LINUX_RMAP_H */