blob: 26eabf5ec718a457eb9cef5635f5c475cca0c48d [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001#ifndef _LINUX_PAGEMAP_H
2#define _LINUX_PAGEMAP_H
3
4/*
5 * Copyright 1995 Linus Torvalds
6 */
7#include <linux/mm.h>
8#include <linux/fs.h>
9#include <linux/list.h>
10#include <linux/highmem.h>
11#include <linux/compiler.h>
12#include <asm/uaccess.h>
13#include <linux/gfp.h>
Guillaume Chazarain3e9f45b2007-05-08 00:23:25 -070014#include <linux/bitops.h>
Nick Piggine2867812008-07-25 19:45:30 -070015#include <linux/hardirq.h> /* for in_interrupt() */
Naoya Horiguchi8edf3442010-05-28 09:29:15 +090016#include <linux/hugetlb_inline.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070017
18/*
19 * Bits in mapping->flags. The lower __GFP_BITS_SHIFT bits are the page
20 * allocation mode flags.
21 */
Lee Schermerhorn9a896c92009-04-02 16:56:45 -070022enum mapping_flags {
23 AS_EIO = __GFP_BITS_SHIFT + 0, /* IO error on async write */
24 AS_ENOSPC = __GFP_BITS_SHIFT + 1, /* ENOSPC on async write */
25 AS_MM_ALL_LOCKS = __GFP_BITS_SHIFT + 2, /* under mm_take_all_locks() */
Lee Schermerhorn9a896c92009-04-02 16:56:45 -070026 AS_UNEVICTABLE = __GFP_BITS_SHIFT + 3, /* e.g., ramdisk, SHM_LOCK */
Konstantin Khlebnikov9d1ba802014-10-09 15:29:29 -070027 AS_EXITING = __GFP_BITS_SHIFT + 4, /* final truncate in progress */
Lee Schermerhorn9a896c92009-04-02 16:56:45 -070028};
Linus Torvalds1da177e2005-04-16 15:20:36 -070029
Guillaume Chazarain3e9f45b2007-05-08 00:23:25 -070030static inline void mapping_set_error(struct address_space *mapping, int error)
31{
Andrew Morton2185e692008-07-23 21:27:19 -070032 if (unlikely(error)) {
Guillaume Chazarain3e9f45b2007-05-08 00:23:25 -070033 if (error == -ENOSPC)
34 set_bit(AS_ENOSPC, &mapping->flags);
35 else
36 set_bit(AS_EIO, &mapping->flags);
37 }
38}
39
Lee Schermerhornba9ddf42008-10-18 20:26:42 -070040static inline void mapping_set_unevictable(struct address_space *mapping)
41{
42 set_bit(AS_UNEVICTABLE, &mapping->flags);
43}
44
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -070045static inline void mapping_clear_unevictable(struct address_space *mapping)
46{
47 clear_bit(AS_UNEVICTABLE, &mapping->flags);
48}
49
Lee Schermerhornba9ddf42008-10-18 20:26:42 -070050static inline int mapping_unevictable(struct address_space *mapping)
51{
Steven Rostedt088e5462011-01-13 15:46:16 -080052 if (mapping)
Lee Schermerhorn89e004ea2008-10-18 20:26:43 -070053 return test_bit(AS_UNEVICTABLE, &mapping->flags);
54 return !!mapping;
Lee Schermerhornba9ddf42008-10-18 20:26:42 -070055}
Lee Schermerhornba9ddf42008-10-18 20:26:42 -070056
Johannes Weiner91b0abe2014-04-03 14:47:49 -070057static inline void mapping_set_exiting(struct address_space *mapping)
58{
59 set_bit(AS_EXITING, &mapping->flags);
60}
61
62static inline int mapping_exiting(struct address_space *mapping)
63{
64 return test_bit(AS_EXITING, &mapping->flags);
65}
66
Al Virodd0fc662005-10-07 07:46:04 +010067static inline gfp_t mapping_gfp_mask(struct address_space * mapping)
Linus Torvalds1da177e2005-04-16 15:20:36 -070068{
Al Viro260b2362005-10-21 03:22:44 -040069 return (__force gfp_t)mapping->flags & __GFP_BITS_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -070070}
71
Michal Hockoc62d2552015-11-06 16:28:49 -080072/* Restricts the given gfp_mask to what the mapping allows. */
73static inline gfp_t mapping_gfp_constraint(struct address_space *mapping,
74 gfp_t gfp_mask)
75{
76 return mapping_gfp_mask(mapping) & gfp_mask;
77}
78
Linus Torvalds1da177e2005-04-16 15:20:36 -070079/*
80 * This is non-atomic. Only to be used before the mapping is activated.
81 * Probably needs a barrier...
82 */
Al Viro260b2362005-10-21 03:22:44 -040083static inline void mapping_set_gfp_mask(struct address_space *m, gfp_t mask)
Linus Torvalds1da177e2005-04-16 15:20:36 -070084{
Al Viro260b2362005-10-21 03:22:44 -040085 m->flags = (m->flags & ~(__force unsigned long)__GFP_BITS_MASK) |
86 (__force unsigned long)mask;
Linus Torvalds1da177e2005-04-16 15:20:36 -070087}
88
89/*
Raymond L. Rivera50d8a182014-07-24 02:39:45 -070090 * The page cache can be done in larger chunks than
Linus Torvalds1da177e2005-04-16 15:20:36 -070091 * one page, because it allows for more efficient
92 * throughput (it can then be mapped into user
93 * space in smaller chunks for same flexibility).
94 *
95 * Or rather, it _will_ be done in larger chunks.
96 */
97#define PAGE_CACHE_SHIFT PAGE_SHIFT
98#define PAGE_CACHE_SIZE PAGE_SIZE
99#define PAGE_CACHE_MASK PAGE_MASK
100#define PAGE_CACHE_ALIGN(addr) (((addr)+PAGE_CACHE_SIZE-1)&PAGE_CACHE_MASK)
101
102#define page_cache_get(page) get_page(page)
103#define page_cache_release(page) put_page(page)
Mel Gormanb745bc82014-06-04 16:10:22 -0700104void release_pages(struct page **pages, int nr, bool cold);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700105
Nick Piggine2867812008-07-25 19:45:30 -0700106/*
107 * speculatively take a reference to a page.
108 * If the page is free (_count == 0), then _count is untouched, and 0
109 * is returned. Otherwise, _count is incremented by 1 and 1 is returned.
110 *
111 * This function must be called inside the same rcu_read_lock() section as has
112 * been used to lookup the page in the pagecache radix-tree (or page table):
113 * this allows allocators to use a synchronize_rcu() to stabilize _count.
114 *
115 * Unless an RCU grace period has passed, the count of all pages coming out
116 * of the allocator must be considered unstable. page_count may return higher
117 * than expected, and put_page must be able to do the right thing when the
118 * page has been finished with, no matter what it is subsequently allocated
119 * for (because put_page is what is used here to drop an invalid speculative
120 * reference).
121 *
122 * This is the interesting part of the lockless pagecache (and lockless
123 * get_user_pages) locking protocol, where the lookup-side (eg. find_get_page)
124 * has the following pattern:
125 * 1. find page in radix tree
126 * 2. conditionally increment refcount
127 * 3. check the page is still in pagecache (if no, goto 1)
128 *
129 * Remove-side that cares about stability of _count (eg. reclaim) has the
130 * following (with tree_lock held for write):
131 * A. atomically check refcount is correct and set it to 0 (atomic_cmpxchg)
132 * B. remove page from pagecache
133 * C. free the page
134 *
135 * There are 2 critical interleavings that matter:
136 * - 2 runs before A: in this case, A sees elevated refcount and bails out
137 * - A runs before 2: in this case, 2 sees zero refcount and retries;
138 * subsequently, B will complete and 1 will find no page, causing the
139 * lookup to return NULL.
140 *
141 * It is possible that between 1 and 2, the page is removed then the exact same
142 * page is inserted into the same position in pagecache. That's OK: the
143 * old find_get_page using tree_lock could equally have run before or after
144 * such a re-insertion, depending on order that locks are granted.
145 *
146 * Lookups racing against pagecache insertion isn't a big problem: either 1
147 * will find the page or it will not. Likewise, the old find_get_page could run
148 * either before the insertion or afterwards, depending on timing.
149 */
150static inline int page_cache_get_speculative(struct page *page)
151{
152 VM_BUG_ON(in_interrupt());
153
Paul E. McKenney8375ad92013-04-29 15:06:13 -0700154#ifdef CONFIG_TINY_RCU
Frederic Weisbeckerbdd4e852011-06-08 01:13:27 +0200155# ifdef CONFIG_PREEMPT_COUNT
Nick Piggine2867812008-07-25 19:45:30 -0700156 VM_BUG_ON(!in_atomic());
157# endif
158 /*
159 * Preempt must be disabled here - we rely on rcu_read_lock doing
160 * this for us.
161 *
162 * Pagecache won't be truncated from interrupt context, so if we have
163 * found a page in the radix tree here, we have pinned its refcount by
164 * disabling preempt, and hence no need for the "speculative get" that
165 * SMP requires.
166 */
Sasha Levin309381fea2014-01-23 15:52:54 -0800167 VM_BUG_ON_PAGE(page_count(page) == 0, page);
Nick Piggine2867812008-07-25 19:45:30 -0700168 atomic_inc(&page->_count);
169
170#else
171 if (unlikely(!get_page_unless_zero(page))) {
172 /*
173 * Either the page has been freed, or will be freed.
174 * In either case, retry here and the caller should
175 * do the right thing (see comments above).
176 */
177 return 0;
178 }
179#endif
Sasha Levin309381fea2014-01-23 15:52:54 -0800180 VM_BUG_ON_PAGE(PageTail(page), page);
Nick Piggine2867812008-07-25 19:45:30 -0700181
182 return 1;
183}
184
Nick Piggince0ad7f2008-07-30 15:23:13 +1000185/*
186 * Same as above, but add instead of inc (could just be merged)
187 */
188static inline int page_cache_add_speculative(struct page *page, int count)
189{
190 VM_BUG_ON(in_interrupt());
191
Paul E. McKenneyb560d8a2009-08-21 22:08:51 -0700192#if !defined(CONFIG_SMP) && defined(CONFIG_TREE_RCU)
Frederic Weisbeckerbdd4e852011-06-08 01:13:27 +0200193# ifdef CONFIG_PREEMPT_COUNT
Nick Piggince0ad7f2008-07-30 15:23:13 +1000194 VM_BUG_ON(!in_atomic());
195# endif
Sasha Levin309381fea2014-01-23 15:52:54 -0800196 VM_BUG_ON_PAGE(page_count(page) == 0, page);
Nick Piggince0ad7f2008-07-30 15:23:13 +1000197 atomic_add(count, &page->_count);
198
199#else
200 if (unlikely(!atomic_add_unless(&page->_count, count, 0)))
201 return 0;
202#endif
Sasha Levin309381fea2014-01-23 15:52:54 -0800203 VM_BUG_ON_PAGE(PageCompound(page) && page != compound_head(page), page);
Nick Piggince0ad7f2008-07-30 15:23:13 +1000204
205 return 1;
206}
207
Nick Piggine2867812008-07-25 19:45:30 -0700208static inline int page_freeze_refs(struct page *page, int count)
209{
210 return likely(atomic_cmpxchg(&page->_count, count, 0) == count);
211}
212
213static inline void page_unfreeze_refs(struct page *page, int count)
214{
Sasha Levin309381fea2014-01-23 15:52:54 -0800215 VM_BUG_ON_PAGE(page_count(page) != 0, page);
Nick Piggine2867812008-07-25 19:45:30 -0700216 VM_BUG_ON(count == 0);
217
218 atomic_set(&page->_count, count);
219}
220
Paul Jackson44110fe2006-03-24 03:16:04 -0800221#ifdef CONFIG_NUMA
Nick Piggin2ae88142006-10-28 10:38:23 -0700222extern struct page *__page_cache_alloc(gfp_t gfp);
Paul Jackson44110fe2006-03-24 03:16:04 -0800223#else
Nick Piggin2ae88142006-10-28 10:38:23 -0700224static inline struct page *__page_cache_alloc(gfp_t gfp)
225{
226 return alloc_pages(gfp, 0);
227}
228#endif
229
Linus Torvalds1da177e2005-04-16 15:20:36 -0700230static inline struct page *page_cache_alloc(struct address_space *x)
231{
Nick Piggin2ae88142006-10-28 10:38:23 -0700232 return __page_cache_alloc(mapping_gfp_mask(x));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233}
234
235static inline struct page *page_cache_alloc_cold(struct address_space *x)
236{
Nick Piggin2ae88142006-10-28 10:38:23 -0700237 return __page_cache_alloc(mapping_gfp_mask(x)|__GFP_COLD);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238}
239
Wu Fengguang7b1de582011-05-24 17:12:25 -0700240static inline struct page *page_cache_alloc_readahead(struct address_space *x)
241{
242 return __page_cache_alloc(mapping_gfp_mask(x) |
243 __GFP_COLD | __GFP_NORETRY | __GFP_NOWARN);
244}
245
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246typedef int filler_t(void *, struct page *);
247
Johannes Weinere7b563b2014-04-03 14:47:44 -0700248pgoff_t page_cache_next_hole(struct address_space *mapping,
249 pgoff_t index, unsigned long max_scan);
250pgoff_t page_cache_prev_hole(struct address_space *mapping,
251 pgoff_t index, unsigned long max_scan);
252
Mel Gorman2457aec2014-06-04 16:10:31 -0700253#define FGP_ACCESSED 0x00000001
254#define FGP_LOCK 0x00000002
255#define FGP_CREAT 0x00000004
256#define FGP_WRITE 0x00000008
257#define FGP_NOFS 0x00000010
258#define FGP_NOWAIT 0x00000020
259
260struct page *pagecache_get_page(struct address_space *mapping, pgoff_t offset,
Michal Hocko45f87de2014-12-29 20:30:35 +0100261 int fgp_flags, gfp_t cache_gfp_mask);
Mel Gorman2457aec2014-06-04 16:10:31 -0700262
263/**
264 * find_get_page - find and get a page reference
265 * @mapping: the address_space to search
266 * @offset: the page index
267 *
268 * Looks up the page cache slot at @mapping & @offset. If there is a
269 * page cache page, it is returned with an increased refcount.
270 *
271 * Otherwise, %NULL is returned.
272 */
273static inline struct page *find_get_page(struct address_space *mapping,
274 pgoff_t offset)
275{
Michal Hocko45f87de2014-12-29 20:30:35 +0100276 return pagecache_get_page(mapping, offset, 0, 0);
Mel Gorman2457aec2014-06-04 16:10:31 -0700277}
278
279static inline struct page *find_get_page_flags(struct address_space *mapping,
280 pgoff_t offset, int fgp_flags)
281{
Michal Hocko45f87de2014-12-29 20:30:35 +0100282 return pagecache_get_page(mapping, offset, fgp_flags, 0);
Mel Gorman2457aec2014-06-04 16:10:31 -0700283}
284
285/**
286 * find_lock_page - locate, pin and lock a pagecache page
287 * pagecache_get_page - find and get a page reference
288 * @mapping: the address_space to search
289 * @offset: the page index
290 *
291 * Looks up the page cache slot at @mapping & @offset. If there is a
292 * page cache page, it is returned locked and with an increased
293 * refcount.
294 *
295 * Otherwise, %NULL is returned.
296 *
297 * find_lock_page() may sleep.
298 */
299static inline struct page *find_lock_page(struct address_space *mapping,
300 pgoff_t offset)
301{
Michal Hocko45f87de2014-12-29 20:30:35 +0100302 return pagecache_get_page(mapping, offset, FGP_LOCK, 0);
Mel Gorman2457aec2014-06-04 16:10:31 -0700303}
304
305/**
306 * find_or_create_page - locate or add a pagecache page
307 * @mapping: the page's address_space
308 * @index: the page's index into the mapping
309 * @gfp_mask: page allocation mode
310 *
311 * Looks up the page cache slot at @mapping & @offset. If there is a
312 * page cache page, it is returned locked and with an increased
313 * refcount.
314 *
315 * If the page is not present, a new page is allocated using @gfp_mask
316 * and added to the page cache and the VM's LRU list. The page is
317 * returned locked and with an increased refcount.
318 *
319 * On memory exhaustion, %NULL is returned.
320 *
321 * find_or_create_page() may sleep, even if @gfp_flags specifies an
322 * atomic allocation!
323 */
324static inline struct page *find_or_create_page(struct address_space *mapping,
325 pgoff_t offset, gfp_t gfp_mask)
326{
327 return pagecache_get_page(mapping, offset,
328 FGP_LOCK|FGP_ACCESSED|FGP_CREAT,
Michal Hocko45f87de2014-12-29 20:30:35 +0100329 gfp_mask);
Mel Gorman2457aec2014-06-04 16:10:31 -0700330}
331
332/**
333 * grab_cache_page_nowait - returns locked page at given index in given cache
334 * @mapping: target address_space
335 * @index: the page index
336 *
337 * Same as grab_cache_page(), but do not wait if the page is unavailable.
338 * This is intended for speculative data generators, where the data can
339 * be regenerated if the page couldn't be grabbed. This routine should
340 * be safe to call while holding the lock for another page.
341 *
342 * Clear __GFP_FS when allocating the page to avoid recursion into the fs
343 * and deadlock against the caller's locked page.
344 */
345static inline struct page *grab_cache_page_nowait(struct address_space *mapping,
346 pgoff_t index)
347{
348 return pagecache_get_page(mapping, index,
349 FGP_LOCK|FGP_CREAT|FGP_NOFS|FGP_NOWAIT,
Michal Hocko45f87de2014-12-29 20:30:35 +0100350 mapping_gfp_mask(mapping));
Mel Gorman2457aec2014-06-04 16:10:31 -0700351}
352
Johannes Weiner0cd61442014-04-03 14:47:46 -0700353struct page *find_get_entry(struct address_space *mapping, pgoff_t offset);
Johannes Weiner0cd61442014-04-03 14:47:46 -0700354struct page *find_lock_entry(struct address_space *mapping, pgoff_t offset);
Johannes Weiner0cd61442014-04-03 14:47:46 -0700355unsigned find_get_entries(struct address_space *mapping, pgoff_t start,
356 unsigned int nr_entries, struct page **entries,
357 pgoff_t *indices);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700358unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
359 unsigned int nr_pages, struct page **pages);
Jens Axboeebf43502006-04-27 08:46:01 +0200360unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t start,
361 unsigned int nr_pages, struct page **pages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
363 int tag, unsigned int nr_pages, struct page **pages);
364
Nick Piggin54566b22009-01-04 12:00:53 -0800365struct page *grab_cache_page_write_begin(struct address_space *mapping,
366 pgoff_t index, unsigned flags);
Nick Pigginafddba42007-10-16 01:25:01 -0700367
Linus Torvalds1da177e2005-04-16 15:20:36 -0700368/*
369 * Returns locked page at given index in given cache, creating it if needed.
370 */
Fengguang Wu57f6b962007-10-16 01:24:37 -0700371static inline struct page *grab_cache_page(struct address_space *mapping,
372 pgoff_t index)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700373{
374 return find_or_create_page(mapping, index, mapping_gfp_mask(mapping));
375}
376
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377extern struct page * read_cache_page(struct address_space *mapping,
Hugh Dickins5e5358e2011-07-25 17:12:23 -0700378 pgoff_t index, filler_t *filler, void *data);
Linus Torvalds0531b2a2010-01-27 09:20:03 -0800379extern struct page * read_cache_page_gfp(struct address_space *mapping,
380 pgoff_t index, gfp_t gfp_mask);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700381extern int read_cache_pages(struct address_space *mapping,
382 struct list_head *pages, filler_t *filler, void *data);
383
Pekka Enberg090d2b12006-06-23 02:05:08 -0700384static inline struct page *read_mapping_page(struct address_space *mapping,
Hugh Dickins5e5358e2011-07-25 17:12:23 -0700385 pgoff_t index, void *data)
Pekka Enberg090d2b12006-06-23 02:05:08 -0700386{
387 filler_t *filler = (filler_t *)mapping->a_ops->readpage;
388 return read_cache_page(mapping, index, filler, data);
389}
390
Nick Piggine2867812008-07-25 19:45:30 -0700391/*
Naoya Horiguchia0f7a752014-07-23 14:00:01 -0700392 * Get the offset in PAGE_SIZE.
393 * (TODO: hugepage should have ->index in PAGE_SIZE)
394 */
395static inline pgoff_t page_to_pgoff(struct page *page)
396{
397 if (unlikely(PageHeadHuge(page)))
398 return page->index << compound_order(page);
399 else
400 return page->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
401}
402
403/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700404 * Return byte-offset into filesystem object for page.
405 */
406static inline loff_t page_offset(struct page *page)
407{
408 return ((loff_t)page->index) << PAGE_CACHE_SHIFT;
409}
410
Mel Gormanf981c592012-07-31 16:44:47 -0700411static inline loff_t page_file_offset(struct page *page)
412{
413 return ((loff_t)page_file_index(page)) << PAGE_CACHE_SHIFT;
414}
415
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +0900416extern pgoff_t linear_hugepage_index(struct vm_area_struct *vma,
417 unsigned long address);
418
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419static inline pgoff_t linear_page_index(struct vm_area_struct *vma,
420 unsigned long address)
421{
Naoya Horiguchi0fe6e202010-05-28 09:29:16 +0900422 pgoff_t pgoff;
423 if (unlikely(is_vm_hugetlb_page(vma)))
424 return linear_hugepage_index(vma, address);
425 pgoff = (address - vma->vm_start) >> PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426 pgoff += vma->vm_pgoff;
427 return pgoff >> (PAGE_CACHE_SHIFT - PAGE_SHIFT);
428}
429
Harvey Harrisonb3c97522008-02-13 15:03:15 -0800430extern void __lock_page(struct page *page);
431extern int __lock_page_killable(struct page *page);
Michel Lespinassed065bd82010-10-26 14:21:57 -0700432extern int __lock_page_or_retry(struct page *page, struct mm_struct *mm,
433 unsigned int flags);
Harvey Harrisonb3c97522008-02-13 15:03:15 -0800434extern void unlock_page(struct page *page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435
Nick Pigginf45840b2008-10-18 20:26:57 -0700436static inline void __set_page_locked(struct page *page)
Nick Piggin529ae9a2008-08-02 12:01:03 +0200437{
Nick Pigginf45840b2008-10-18 20:26:57 -0700438 __set_bit(PG_locked, &page->flags);
Nick Piggin529ae9a2008-08-02 12:01:03 +0200439}
440
Nick Pigginf45840b2008-10-18 20:26:57 -0700441static inline void __clear_page_locked(struct page *page)
Nick Piggin529ae9a2008-08-02 12:01:03 +0200442{
Nick Pigginf45840b2008-10-18 20:26:57 -0700443 __clear_bit(PG_locked, &page->flags);
Nick Piggin529ae9a2008-08-02 12:01:03 +0200444}
445
446static inline int trylock_page(struct page *page)
447{
Nick Piggin8413ac92008-10-18 20:26:59 -0700448 return (likely(!test_and_set_bit_lock(PG_locked, &page->flags)));
Nick Piggin529ae9a2008-08-02 12:01:03 +0200449}
450
Nick Piggindb376482006-09-25 23:31:24 -0700451/*
452 * lock_page may only be called if we have the page's inode pinned.
453 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700454static inline void lock_page(struct page *page)
455{
456 might_sleep();
Nick Piggin529ae9a2008-08-02 12:01:03 +0200457 if (!trylock_page(page))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700458 __lock_page(page);
459}
Nick Piggindb376482006-09-25 23:31:24 -0700460
461/*
Matthew Wilcox2687a352007-12-06 11:18:49 -0500462 * lock_page_killable is like lock_page but can be interrupted by fatal
463 * signals. It returns 0 if it locked the page and -EINTR if it was
464 * killed while waiting.
465 */
466static inline int lock_page_killable(struct page *page)
467{
468 might_sleep();
Nick Piggin529ae9a2008-08-02 12:01:03 +0200469 if (!trylock_page(page))
Matthew Wilcox2687a352007-12-06 11:18:49 -0500470 return __lock_page_killable(page);
471 return 0;
472}
473
474/*
Michel Lespinassed065bd82010-10-26 14:21:57 -0700475 * lock_page_or_retry - Lock the page, unless this would block and the
476 * caller indicated that it can handle a retry.
Paul Cassella9a95f3c2014-08-06 16:07:24 -0700477 *
478 * Return value and mmap_sem implications depend on flags; see
479 * __lock_page_or_retry().
Michel Lespinassed065bd82010-10-26 14:21:57 -0700480 */
481static inline int lock_page_or_retry(struct page *page, struct mm_struct *mm,
482 unsigned int flags)
483{
484 might_sleep();
485 return trylock_page(page) || __lock_page_or_retry(page, mm, flags);
486}
487
488/*
NeilBrowna4796e32014-09-24 11:28:32 +1000489 * This is exported only for wait_on_page_locked/wait_on_page_writeback,
490 * and for filesystems which need to wait on PG_private.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700491 */
Harvey Harrisonb3c97522008-02-13 15:03:15 -0800492extern void wait_on_page_bit(struct page *page, int bit_nr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700493
KOSAKI Motohirof62e00c2011-05-24 17:11:29 -0700494extern int wait_on_page_bit_killable(struct page *page, int bit_nr);
NeilBrowncbbce822014-09-25 13:55:19 +1000495extern int wait_on_page_bit_killable_timeout(struct page *page,
496 int bit_nr, unsigned long timeout);
KOSAKI Motohirof62e00c2011-05-24 17:11:29 -0700497
498static inline int wait_on_page_locked_killable(struct page *page)
499{
500 if (PageLocked(page))
501 return wait_on_page_bit_killable(page, PG_locked);
502 return 0;
503}
504
NeilBrowna4796e32014-09-24 11:28:32 +1000505extern wait_queue_head_t *page_waitqueue(struct page *page);
506static inline void wake_up_page(struct page *page, int bit)
507{
508 __wake_up_bit(page_waitqueue(page), &page->flags, bit);
509}
510
Linus Torvalds1da177e2005-04-16 15:20:36 -0700511/*
512 * Wait for a page to be unlocked.
513 *
514 * This must be called with the caller "holding" the page,
515 * ie with increased "page->count" so that the page won't
516 * go away during the wait..
517 */
518static inline void wait_on_page_locked(struct page *page)
519{
520 if (PageLocked(page))
521 wait_on_page_bit(page, PG_locked);
522}
523
524/*
525 * Wait for a page to complete writeback
526 */
527static inline void wait_on_page_writeback(struct page *page)
528{
529 if (PageWriteback(page))
530 wait_on_page_bit(page, PG_writeback);
531}
532
533extern void end_page_writeback(struct page *page);
Darrick J. Wong1d1d1a72013-02-21 16:42:51 -0800534void wait_for_stable_page(struct page *page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700535
Matthew Wilcox57d99842014-06-04 16:07:45 -0700536void page_endio(struct page *page, int rw, int err);
537
Linus Torvalds1da177e2005-04-16 15:20:36 -0700538/*
David Howells385e1ca5f2009-04-03 16:42:39 +0100539 * Add an arbitrary waiter to a page's wait queue
540 */
541extern void add_page_wait_queue(struct page *page, wait_queue_t *waiter);
542
543/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700544 * Fault a userspace page into pagetables. Return non-zero on a fault.
545 *
546 * This assumes that two userspace pages are always sufficient. That's
547 * not true if PAGE_CACHE_SIZE > PAGE_SIZE.
548 */
549static inline int fault_in_pages_writeable(char __user *uaddr, int size)
550{
551 int ret;
552
Nick Piggin08291422007-10-16 01:24:59 -0700553 if (unlikely(size == 0))
554 return 0;
555
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556 /*
557 * Writing zeroes into userspace here is OK, because we know that if
558 * the zero gets there, we'll be overwriting it.
559 */
560 ret = __put_user(0, uaddr);
561 if (ret == 0) {
562 char __user *end = uaddr + size - 1;
563
564 /*
565 * If the page was already mapped, this will get a cache miss
566 * for sure, so try to avoid doing it.
567 */
568 if (((unsigned long)uaddr & PAGE_MASK) !=
569 ((unsigned long)end & PAGE_MASK))
Daniel Vetterf56f8212012-03-25 19:47:41 +0200570 ret = __put_user(0, end);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700571 }
572 return ret;
573}
574
Nick Piggin08291422007-10-16 01:24:59 -0700575static inline int fault_in_pages_readable(const char __user *uaddr, int size)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700576{
577 volatile char c;
578 int ret;
579
Nick Piggin08291422007-10-16 01:24:59 -0700580 if (unlikely(size == 0))
581 return 0;
582
Linus Torvalds1da177e2005-04-16 15:20:36 -0700583 ret = __get_user(c, uaddr);
584 if (ret == 0) {
585 const char __user *end = uaddr + size - 1;
586
587 if (((unsigned long)uaddr & PAGE_MASK) !=
Andi Kleen627295e2010-08-09 17:19:02 -0700588 ((unsigned long)end & PAGE_MASK)) {
Daniel Vetterf56f8212012-03-25 19:47:41 +0200589 ret = __get_user(c, end);
Andi Kleen627295e2010-08-09 17:19:02 -0700590 (void)c;
591 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700592 }
Nick Piggin08291422007-10-16 01:24:59 -0700593 return ret;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700594}
595
Daniel Vetterf56f8212012-03-25 19:47:41 +0200596/*
597 * Multipage variants of the above prefault helpers, useful if more than
598 * PAGE_SIZE of data needs to be prefaulted. These are separate from the above
599 * functions (which only handle up to PAGE_SIZE) to avoid clobbering the
600 * filemap.c hotpaths.
601 */
602static inline int fault_in_multipages_writeable(char __user *uaddr, int size)
603{
Paul Gortmakeraf2e8402012-05-29 15:06:14 -0700604 int ret = 0;
Daniel Vetter99237772012-04-14 18:03:10 +0200605 char __user *end = uaddr + size - 1;
Daniel Vetterf56f8212012-03-25 19:47:41 +0200606
607 if (unlikely(size == 0))
Paul Gortmakeraf2e8402012-05-29 15:06:14 -0700608 return ret;
Daniel Vetterf56f8212012-03-25 19:47:41 +0200609
610 /*
611 * Writing zeroes into userspace here is OK, because we know that if
612 * the zero gets there, we'll be overwriting it.
613 */
614 while (uaddr <= end) {
615 ret = __put_user(0, uaddr);
616 if (ret != 0)
617 return ret;
618 uaddr += PAGE_SIZE;
619 }
620
621 /* Check whether the range spilled into the next page. */
622 if (((unsigned long)uaddr & PAGE_MASK) ==
623 ((unsigned long)end & PAGE_MASK))
624 ret = __put_user(0, end);
625
626 return ret;
627}
628
629static inline int fault_in_multipages_readable(const char __user *uaddr,
630 int size)
631{
632 volatile char c;
Paul Gortmakeraf2e8402012-05-29 15:06:14 -0700633 int ret = 0;
Daniel Vetterf56f8212012-03-25 19:47:41 +0200634 const char __user *end = uaddr + size - 1;
635
636 if (unlikely(size == 0))
Paul Gortmakeraf2e8402012-05-29 15:06:14 -0700637 return ret;
Daniel Vetterf56f8212012-03-25 19:47:41 +0200638
639 while (uaddr <= end) {
640 ret = __get_user(c, uaddr);
641 if (ret != 0)
642 return ret;
643 uaddr += PAGE_SIZE;
644 }
645
646 /* Check whether the range spilled into the next page. */
647 if (((unsigned long)uaddr & PAGE_MASK) ==
648 ((unsigned long)end & PAGE_MASK)) {
649 ret = __get_user(c, end);
650 (void)c;
651 }
652
653 return ret;
654}
655
Nick Piggin529ae9a2008-08-02 12:01:03 +0200656int add_to_page_cache_locked(struct page *page, struct address_space *mapping,
657 pgoff_t index, gfp_t gfp_mask);
658int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
659 pgoff_t index, gfp_t gfp_mask);
Minchan Kim97cecb52011-03-22 16:30:53 -0700660extern void delete_from_page_cache(struct page *page);
Greg Thelenc4843a72015-05-22 17:13:16 -0400661extern void __delete_from_page_cache(struct page *page, void *shadow,
662 struct mem_cgroup *memcg);
Miklos Szeredief6a3c62011-03-22 16:30:52 -0700663int replace_page_cache_page(struct page *old, struct page *new, gfp_t gfp_mask);
Nick Piggin529ae9a2008-08-02 12:01:03 +0200664
665/*
666 * Like add_to_page_cache_locked, but used to add newly allocated pages:
Nick Pigginf45840b2008-10-18 20:26:57 -0700667 * the page is new, so we can just run __set_page_locked() against it.
Nick Piggin529ae9a2008-08-02 12:01:03 +0200668 */
669static inline int add_to_page_cache(struct page *page,
670 struct address_space *mapping, pgoff_t offset, gfp_t gfp_mask)
671{
672 int error;
673
Nick Pigginf45840b2008-10-18 20:26:57 -0700674 __set_page_locked(page);
Nick Piggin529ae9a2008-08-02 12:01:03 +0200675 error = add_to_page_cache_locked(page, mapping, offset, gfp_mask);
676 if (unlikely(error))
Nick Pigginf45840b2008-10-18 20:26:57 -0700677 __clear_page_locked(page);
Nick Piggin529ae9a2008-08-02 12:01:03 +0200678 return error;
679}
680
Fabian Frederickb57c2cb2015-05-24 17:19:41 +0200681static inline unsigned long dir_pages(struct inode *inode)
682{
683 return (unsigned long)(inode->i_size + PAGE_CACHE_SIZE - 1) >>
684 PAGE_CACHE_SHIFT;
685}
686
Linus Torvalds1da177e2005-04-16 15:20:36 -0700687#endif /* _LINUX_PAGEMAP_H */