blob: 49eb4f8ebac9636a394dbe097532c70a08e056d7 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef _LINUX_RMAP_H
2#define _LINUX_RMAP_H
3/*
4 * Declarations for Reverse Mapping functions in mm/rmap.c
5 */
6
Linus Torvalds1da177e2005-04-16 15:20:36 -07007#include <linux/list.h>
8#include <linux/slab.h>
9#include <linux/mm.h>
Ingo Molnar5a505082012-12-02 19:56:46 +000010#include <linux/rwsem.h>
Balbir Singhbed71612008-02-07 00:14:01 -080011#include <linux/memcontrol.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070012
13/*
14 * The anon_vma heads a list of private "related" vmas, to scan if
15 * an anonymous page pointing to this anon_vma needs to be unmapped:
16 * the vmas on the list will be related by forking, or by splitting.
17 *
18 * Since vmas come and go as they are split and merged (particularly
19 * in mprotect), the mapping field of an anonymous page cannot point
20 * directly to a vma: instead it points to an anon_vma, on whose list
21 * the related vmas can be easily linked or unlinked.
22 *
23 * After unlinking the last vma on the list, we must garbage collect
24 * the anon_vma object itself: we're guaranteed no page can be
25 * pointing to this anon_vma once its vma list is empty.
26 */
27struct anon_vma {
Ingo Molnar5a505082012-12-02 19:56:46 +000028 struct anon_vma *root; /* Root of this anon_vma tree */
29 struct rw_semaphore rwsem; /* W: modification, R: walking the list */
Mel Gorman7f60c212010-05-24 14:32:18 -070030 /*
Peter Zijlstra83813262011-03-22 16:32:48 -070031 * The refcount is taken on an anon_vma when there is no
Mel Gorman7f60c212010-05-24 14:32:18 -070032 * guarantee that the vma of page tables will exist for
33 * the duration of the operation. A caller that takes
34 * the reference is responsible for clearing up the
35 * anon_vma if they are the last user on release
36 */
Peter Zijlstra83813262011-03-22 16:32:48 -070037 atomic_t refcount;
38
Andrea Arcangeli7906d002008-07-28 15:46:26 -070039 /*
Konstantin Khlebnikov7a3ef202015-01-08 14:32:15 -080040 * Count of child anon_vmas and VMAs which points to this anon_vma.
41 *
42 * This counter is used for making decision about reusing anon_vma
43 * instead of forking new one. See comments in function anon_vma_clone.
44 */
45 unsigned degree;
46
47 struct anon_vma *parent; /* Parent of this anon_vma */
48
49 /*
Michel Lespinassebf181b92012-10-08 16:31:39 -070050 * NOTE: the LSB of the rb_root.rb_node is set by
Andrea Arcangeli7906d002008-07-28 15:46:26 -070051 * mm_take_all_locks() _after_ taking the above lock. So the
Michel Lespinassebf181b92012-10-08 16:31:39 -070052 * rb_root must only be read/written after taking the above lock
Andrea Arcangeli7906d002008-07-28 15:46:26 -070053 * to be sure to see a valid next pointer. The LSB bit itself
54 * is serialized by a system wide lock only visible to
55 * mm_take_all_locks() (mm_all_locks_mutex).
56 */
Michel Lespinassebf181b92012-10-08 16:31:39 -070057 struct rb_root rb_root; /* Interval tree of private "related" vmas */
Rik van Riel5beb4932010-03-05 13:42:07 -080058};
59
60/*
61 * The copy-on-write semantics of fork mean that an anon_vma
62 * can become associated with multiple processes. Furthermore,
63 * each child process will have its own anon_vma, where new
64 * pages for that process are instantiated.
65 *
66 * This structure allows us to find the anon_vmas associated
67 * with a VMA, or the VMAs associated with an anon_vma.
68 * The "same_vma" list contains the anon_vma_chains linking
69 * all the anon_vmas associated with this VMA.
Michel Lespinassebf181b92012-10-08 16:31:39 -070070 * The "rb" field indexes on an interval tree the anon_vma_chains
Rik van Riel5beb4932010-03-05 13:42:07 -080071 * which link all the VMAs associated with this anon_vma.
72 */
73struct anon_vma_chain {
74 struct vm_area_struct *vma;
75 struct anon_vma *anon_vma;
76 struct list_head same_vma; /* locked by mmap_sem & page_table_lock */
Ingo Molnar5a505082012-12-02 19:56:46 +000077 struct rb_node rb; /* locked by anon_vma->rwsem */
Michel Lespinassebf181b92012-10-08 16:31:39 -070078 unsigned long rb_subtree_last;
Michel Lespinasseed8ea812012-10-08 16:31:45 -070079#ifdef CONFIG_DEBUG_VM_RB
80 unsigned long cached_vma_start, cached_vma_last;
81#endif
Linus Torvalds1da177e2005-04-16 15:20:36 -070082};
83
Minchan Kim02c6de82012-10-08 16:31:55 -070084enum ttu_flags {
Konstantin Khlebnikovdaa5ba72014-06-04 16:10:52 -070085 TTU_UNMAP = 1, /* unmap mode */
86 TTU_MIGRATION = 2, /* migration mode */
87 TTU_MUNLOCK = 4, /* munlock mode */
Minchan Kim854e9ed2016-01-15 16:54:53 -080088 TTU_LZFREE = 8, /* lazy free mode */
Kirill A. Shutemov2a52bcb2016-03-17 14:20:04 -070089 TTU_SPLIT_HUGE_PMD = 16, /* split huge PMD if any */
Minchan Kim02c6de82012-10-08 16:31:55 -070090
91 TTU_IGNORE_MLOCK = (1 << 8), /* ignore mlock */
92 TTU_IGNORE_ACCESS = (1 << 9), /* don't age */
93 TTU_IGNORE_HWPOISON = (1 << 10),/* corrupted page is recoverable */
Mel Gorman72b252a2015-09-04 15:47:32 -070094 TTU_BATCH_FLUSH = (1 << 11), /* Batch TLB flushes where possible
95 * and caller guarantees they will
96 * do a final flush if necessary */
Kirill A. Shutemov2a52bcb2016-03-17 14:20:04 -070097 TTU_RMAP_LOCKED = (1 << 12) /* do not grab rmap lock:
98 * caller holds it */
Minchan Kim02c6de82012-10-08 16:31:55 -070099};
100
Linus Torvalds1da177e2005-04-16 15:20:36 -0700101#ifdef CONFIG_MMU
Rik van Riel76545062010-08-09 17:18:41 -0700102static inline void get_anon_vma(struct anon_vma *anon_vma)
103{
Peter Zijlstra83813262011-03-22 16:32:48 -0700104 atomic_inc(&anon_vma->refcount);
Rik van Riel76545062010-08-09 17:18:41 -0700105}
106
Peter Zijlstra01d8b202011-03-22 16:32:49 -0700107void __put_anon_vma(struct anon_vma *anon_vma);
108
109static inline void put_anon_vma(struct anon_vma *anon_vma)
110{
111 if (atomic_dec_and_test(&anon_vma->refcount))
112 __put_anon_vma(anon_vma);
113}
Linus Torvalds1da177e2005-04-16 15:20:36 -0700114
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +0000115static inline void anon_vma_lock_write(struct anon_vma *anon_vma)
Rik van Rielcba48b92010-08-09 17:18:38 -0700116{
Ingo Molnar5a505082012-12-02 19:56:46 +0000117 down_write(&anon_vma->root->rwsem);
Rik van Rielcba48b92010-08-09 17:18:38 -0700118}
119
Konstantin Khlebnikov08b52702013-02-22 16:34:40 -0800120static inline void anon_vma_unlock_write(struct anon_vma *anon_vma)
Rik van Rielcba48b92010-08-09 17:18:38 -0700121{
Ingo Molnar5a505082012-12-02 19:56:46 +0000122 up_write(&anon_vma->root->rwsem);
Rik van Rielcba48b92010-08-09 17:18:38 -0700123}
124
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +0000125static inline void anon_vma_lock_read(struct anon_vma *anon_vma)
126{
127 down_read(&anon_vma->root->rwsem);
128}
129
130static inline void anon_vma_unlock_read(struct anon_vma *anon_vma)
131{
132 up_read(&anon_vma->root->rwsem);
133}
134
135
Linus Torvalds1da177e2005-04-16 15:20:36 -0700136/*
137 * anon_vma helper functions.
138 */
139void anon_vma_init(void); /* create anon_vma_cachep */
140int anon_vma_prepare(struct vm_area_struct *);
Rik van Riel5beb4932010-03-05 13:42:07 -0800141void unlink_anon_vmas(struct vm_area_struct *);
142int anon_vma_clone(struct vm_area_struct *, struct vm_area_struct *);
143int anon_vma_fork(struct vm_area_struct *, struct vm_area_struct *);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144
Rik van Riel5beb4932010-03-05 13:42:07 -0800145static inline void anon_vma_merge(struct vm_area_struct *vma,
146 struct vm_area_struct *next)
147{
Sasha Levin81d1b092014-10-09 15:28:10 -0700148 VM_BUG_ON_VMA(vma->anon_vma != next->anon_vma, vma);
Rik van Riel5beb4932010-03-05 13:42:07 -0800149 unlink_anon_vmas(next);
150}
151
Peter Zijlstra01d8b202011-03-22 16:32:49 -0700152struct anon_vma *page_get_anon_vma(struct page *page);
153
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -0800154/* bitflags for do_page_add_anon_rmap() */
155#define RMAP_EXCLUSIVE 0x01
156#define RMAP_COMPOUND 0x02
157
Linus Torvalds1da177e2005-04-16 15:20:36 -0700158/*
159 * rmap interfaces called when adding or removing pte of page
160 */
Rik van Rielc44b6742010-03-05 13:42:09 -0800161void page_move_anon_rmap(struct page *, struct vm_area_struct *, unsigned long);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -0800162void page_add_anon_rmap(struct page *, struct vm_area_struct *,
163 unsigned long, bool);
Rik van Rielad8c2ee2010-08-09 17:19:48 -0700164void do_page_add_anon_rmap(struct page *, struct vm_area_struct *,
165 unsigned long, int);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -0800166void page_add_new_anon_rmap(struct page *, struct vm_area_struct *,
167 unsigned long, bool);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700168void page_add_file_rmap(struct page *);
Kirill A. Shutemovd281ee62016-01-15 16:52:16 -0800169void page_remove_rmap(struct page *, bool);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +0900171void hugepage_add_anon_rmap(struct page *, struct vm_area_struct *,
172 unsigned long);
173void hugepage_add_new_anon_rmap(struct page *, struct vm_area_struct *,
174 unsigned long);
175
Kirill A. Shutemov53f92632016-01-15 16:53:42 -0800176static inline void page_dup_rmap(struct page *page, bool compound)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700177{
Kirill A. Shutemov53f92632016-01-15 16:53:42 -0800178 atomic_inc(compound ? compound_mapcount_ptr(page) : &page->_mapcount);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179}
180
181/*
182 * Called from mm/vmscan.c to handle paging out
183 */
Wu Fengguang6fe6b7e2009-06-16 15:33:05 -0700184int page_referenced(struct page *, int is_locked,
Johannes Weiner72835c82012-01-12 17:18:32 -0800185 struct mem_cgroup *memcg, unsigned long *vm_flags);
Hugh Dickins5ad64682009-12-14 17:59:24 -0800186
Andi Kleen14fa31b2009-09-16 11:50:10 +0200187#define TTU_ACTION(x) ((x) & TTU_ACTION_MASK)
188
189int try_to_unmap(struct page *, enum ttu_flags flags);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700190
191/*
Matthew Wilcoxe748dcd2015-02-16 15:59:12 -0800192 * Used by uprobes to replace a userspace page safely
Carsten Otteceffc072005-06-23 22:05:25 -0700193 */
Namhyung Kime9a81a82010-10-26 14:22:01 -0700194pte_t *__page_check_address(struct page *, struct mm_struct *,
Nick Piggin479db0b2008-08-20 14:09:18 -0700195 unsigned long, spinlock_t **, int);
Carsten Otteceffc072005-06-23 22:05:25 -0700196
Namhyung Kime9a81a82010-10-26 14:22:01 -0700197static inline pte_t *page_check_address(struct page *page, struct mm_struct *mm,
198 unsigned long address,
199 spinlock_t **ptlp, int sync)
200{
201 pte_t *ptep;
202
203 __cond_lock(*ptlp, ptep = __page_check_address(page, mm, address,
204 ptlp, sync));
205 return ptep;
206}
207
Carsten Otteceffc072005-06-23 22:05:25 -0700208/*
Vladimir Davydov8749cfe2016-01-15 16:54:45 -0800209 * Used by idle page tracking to check if a page was referenced via page
210 * tables.
211 */
212#ifdef CONFIG_TRANSPARENT_HUGEPAGE
213bool page_check_address_transhuge(struct page *page, struct mm_struct *mm,
214 unsigned long address, pmd_t **pmdp,
215 pte_t **ptep, spinlock_t **ptlp);
216#else
217static inline bool page_check_address_transhuge(struct page *page,
218 struct mm_struct *mm, unsigned long address,
219 pmd_t **pmdp, pte_t **ptep, spinlock_t **ptlp)
220{
221 *ptep = page_check_address(page, mm, address, ptlp, 0);
222 *pmdp = NULL;
223 return !!*ptep;
224}
225#endif
226
227/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228 * Used by swapoff to help locate where page is expected in vma.
229 */
230unsigned long page_address_in_vma(struct page *, struct vm_area_struct *);
231
Peter Zijlstrad08b3852006-09-25 23:30:57 -0700232/*
233 * Cleans the PTEs of shared mappings.
234 * (and since clean PTEs should also be readonly, write protects them too)
235 *
236 * returns the number of cleaned PTEs.
237 */
238int page_mkclean(struct page *);
239
Nick Pigginb291f002008-10-18 20:26:44 -0700240/*
241 * called in munlock()/munmap() path to check for other vmas holding
242 * the page mlocked.
243 */
244int try_to_munlock(struct page *);
Nick Pigginb291f002008-10-18 20:26:44 -0700245
Kirill A. Shutemove3884662016-03-17 14:20:07 -0700246void remove_migration_ptes(struct page *old, struct page *new, bool locked);
247
Andi Kleen10be22d2009-09-16 11:50:04 +0200248/*
249 * Called by memory-failure.c to kill processes.
250 */
Ingo Molnar4fc3f1d2012-12-02 19:56:50 +0000251struct anon_vma *page_lock_anon_vma_read(struct page *page);
252void page_unlock_anon_vma_read(struct anon_vma *anon_vma);
Andi Kleen6a460792009-09-16 11:50:15 +0200253int page_mapped_in_vma(struct page *page, struct vm_area_struct *vma);
Andi Kleen10be22d2009-09-16 11:50:04 +0200254
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -0800255/*
256 * rmap_walk_control: To control rmap traversing for specific needs
257 *
258 * arg: passed to rmap_one() and invalid_vma()
259 * rmap_one: executed on each vma where page is mapped
260 * done: for checking traversing termination condition
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -0800261 * anon_lock: for getting anon_lock by optimized way rather than default
262 * invalid_vma: for skipping uninterested vma
263 */
Joonsoo Kim051ac832014-01-21 15:49:48 -0800264struct rmap_walk_control {
265 void *arg;
266 int (*rmap_one)(struct page *page, struct vm_area_struct *vma,
267 unsigned long addr, void *arg);
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -0800268 int (*done)(struct page *page);
Joonsoo Kim0dd1c7b2014-01-21 15:49:49 -0800269 struct anon_vma *(*anon_lock)(struct page *page);
270 bool (*invalid_vma)(struct vm_area_struct *vma, void *arg);
Joonsoo Kim051ac832014-01-21 15:49:48 -0800271};
272
Joonsoo Kim051ac832014-01-21 15:49:48 -0800273int rmap_walk(struct page *page, struct rmap_walk_control *rwc);
Kirill A. Shutemovb9773192016-03-17 14:20:01 -0700274int rmap_walk_locked(struct page *page, struct rmap_walk_control *rwc);
Hugh Dickinse9995ef2009-12-14 17:59:31 -0800275
Linus Torvalds1da177e2005-04-16 15:20:36 -0700276#else /* !CONFIG_MMU */
277
278#define anon_vma_init() do {} while (0)
279#define anon_vma_prepare(vma) (0)
280#define anon_vma_link(vma) do {} while (0)
281
Mike Frysinger01ff53f2009-06-23 12:37:01 -0700282static inline int page_referenced(struct page *page, int is_locked,
Johannes Weiner72835c82012-01-12 17:18:32 -0800283 struct mem_cgroup *memcg,
Mike Frysinger01ff53f2009-06-23 12:37:01 -0700284 unsigned long *vm_flags)
285{
286 *vm_flags = 0;
Johannes Weiner645747462010-03-05 13:42:22 -0800287 return 0;
Mike Frysinger01ff53f2009-06-23 12:37:01 -0700288}
289
Christoph Lametera48d07a2006-02-01 03:05:38 -0800290#define try_to_unmap(page, refs) SWAP_FAIL
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291
Peter Zijlstrad08b3852006-09-25 23:30:57 -0700292static inline int page_mkclean(struct page *page)
293{
294 return 0;
295}
296
297
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298#endif /* CONFIG_MMU */
299
300/*
301 * Return values of try_to_unmap
302 */
303#define SWAP_SUCCESS 0
304#define SWAP_AGAIN 1
305#define SWAP_FAIL 2
Nick Pigginb291f002008-10-18 20:26:44 -0700306#define SWAP_MLOCK 3
Minchan Kim854e9ed2016-01-15 16:54:53 -0800307#define SWAP_LZFREE 4
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308
309#endif /* _LINUX_RMAP_H */