blob: 014defd2e744e00b4df273e88d2a7026a3db4ff9 [file] [log] [blame]
Matthew Wilcoxd475c632015-02-16 15:58:56 -08001/*
2 * fs/dax.c - Direct Access filesystem code
3 * Copyright (c) 2013-2014 Intel Corporation
4 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
5 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 */
16
17#include <linux/atomic.h>
18#include <linux/blkdev.h>
19#include <linux/buffer_head.h>
Ross Zwislerd77e92e2015-09-09 10:29:40 -060020#include <linux/dax.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080021#include <linux/fs.h>
22#include <linux/genhd.h>
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -080023#include <linux/highmem.h>
24#include <linux/memcontrol.h>
25#include <linux/mm.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080026#include <linux/mutex.h>
Ross Zwisler9973c982016-01-22 15:10:47 -080027#include <linux/pagevec.h>
Ross Zwisler2765cfb2015-08-18 13:55:40 -060028#include <linux/pmem.h>
Matthew Wilcox289c6ae2015-02-16 15:58:59 -080029#include <linux/sched.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080030#include <linux/uio.h>
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -080031#include <linux/vmstat.h>
Dan Williams34c0fd52016-01-15 16:56:14 -080032#include <linux/pfn_t.h>
Dan Williams0e749e52016-01-15 16:55:53 -080033#include <linux/sizes.h>
Christoph Hellwiga254e562016-09-19 11:24:49 +100034#include <linux/iomap.h>
35#include "internal.h"
Matthew Wilcoxd475c632015-02-16 15:58:56 -080036
Jan Karae8043152016-05-12 18:29:16 +020037/*
38 * We use lowest available bit in exceptional entry for locking, other two
39 * bits to determine entry type. In total 3 special bits.
40 */
41#define RADIX_DAX_SHIFT (RADIX_TREE_EXCEPTIONAL_SHIFT + 3)
42#define RADIX_DAX_PTE (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 1))
43#define RADIX_DAX_PMD (1 << (RADIX_TREE_EXCEPTIONAL_SHIFT + 2))
44#define RADIX_DAX_TYPE_MASK (RADIX_DAX_PTE | RADIX_DAX_PMD)
45#define RADIX_DAX_TYPE(entry) ((unsigned long)entry & RADIX_DAX_TYPE_MASK)
NeilBrown78a9be02016-05-20 17:03:51 -070046#define RADIX_DAX_SECTOR(entry) (((unsigned long)entry >> RADIX_DAX_SHIFT))
47#define RADIX_DAX_ENTRY(sector, pmd) ((void *)((unsigned long)sector << \
Jan Karae8043152016-05-12 18:29:16 +020048 RADIX_DAX_SHIFT | (pmd ? RADIX_DAX_PMD : RADIX_DAX_PTE) | \
49 RADIX_TREE_EXCEPTIONAL_ENTRY))
NeilBrowne4b27492016-05-11 11:58:47 +020050
Jan Karaac401cc2016-05-12 18:29:18 +020051/* We choose 4096 entries - same as per-zone page wait tables */
52#define DAX_WAIT_TABLE_BITS 12
53#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)
54
55wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
56
57static int __init init_dax_wait_table(void)
58{
59 int i;
60
61 for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
62 init_waitqueue_head(wait_table + i);
63 return 0;
64}
65fs_initcall(init_dax_wait_table);
66
67static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping,
68 pgoff_t index)
69{
70 unsigned long hash = hash_long((unsigned long)mapping ^ index,
71 DAX_WAIT_TABLE_BITS);
72 return wait_table + hash;
73}
NeilBrown78a9be02016-05-20 17:03:51 -070074
Dan Williamsb2e0d162016-01-15 16:55:59 -080075static long dax_map_atomic(struct block_device *bdev, struct blk_dax_ctl *dax)
76{
77 struct request_queue *q = bdev->bd_queue;
78 long rc = -EIO;
79
Dan Williams7a9eb202016-06-03 18:06:47 -070080 dax->addr = ERR_PTR(-EIO);
Dan Williamsb2e0d162016-01-15 16:55:59 -080081 if (blk_queue_enter(q, true) != 0)
82 return rc;
83
84 rc = bdev_direct_access(bdev, dax);
85 if (rc < 0) {
Dan Williams7a9eb202016-06-03 18:06:47 -070086 dax->addr = ERR_PTR(rc);
Dan Williamsb2e0d162016-01-15 16:55:59 -080087 blk_queue_exit(q);
88 return rc;
89 }
90 return rc;
91}
92
93static void dax_unmap_atomic(struct block_device *bdev,
94 const struct blk_dax_ctl *dax)
95{
96 if (IS_ERR(dax->addr))
97 return;
98 blk_queue_exit(bdev->bd_queue);
99}
100
Dan Williamsd1a5f2b42016-01-28 20:25:31 -0800101struct page *read_dax_sector(struct block_device *bdev, sector_t n)
102{
103 struct page *page = alloc_pages(GFP_KERNEL, 0);
104 struct blk_dax_ctl dax = {
105 .size = PAGE_SIZE,
106 .sector = n & ~((((int) PAGE_SIZE) / 512) - 1),
107 };
108 long rc;
109
110 if (!page)
111 return ERR_PTR(-ENOMEM);
112
113 rc = dax_map_atomic(bdev, &dax);
114 if (rc < 0)
115 return ERR_PTR(rc);
116 memcpy_from_pmem(page_address(page), dax.addr, PAGE_SIZE);
117 dax_unmap_atomic(bdev, &dax);
118 return page;
119}
120
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800121static bool buffer_written(struct buffer_head *bh)
122{
123 return buffer_mapped(bh) && !buffer_unwritten(bh);
124}
125
126/*
127 * When ext4 encounters a hole, it returns without modifying the buffer_head
128 * which means that we can't trust b_size. To cope with this, we set b_state
129 * to 0 before calling get_block and, if any bit is set, we know we can trust
130 * b_size. Unfortunate, really, since ext4 knows precisely how long a hole is
131 * and would save us time calling get_block repeatedly.
132 */
133static bool buffer_size_valid(struct buffer_head *bh)
134{
135 return bh->b_state != 0;
136}
137
Dan Williamsb2e0d162016-01-15 16:55:59 -0800138
139static sector_t to_sector(const struct buffer_head *bh,
140 const struct inode *inode)
141{
142 sector_t sector = bh->b_blocknr << (inode->i_blkbits - 9);
143
144 return sector;
145}
146
Omar Sandovala95cd632015-03-16 04:33:51 -0700147static ssize_t dax_io(struct inode *inode, struct iov_iter *iter,
148 loff_t start, loff_t end, get_block_t get_block,
149 struct buffer_head *bh)
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800150{
Dan Williamsb2e0d162016-01-15 16:55:59 -0800151 loff_t pos = start, max = start, bh_max = start;
Dan Williams14df6a42016-06-01 21:03:32 -0700152 bool hole = false;
Dan Williamsb2e0d162016-01-15 16:55:59 -0800153 struct block_device *bdev = NULL;
154 int rw = iov_iter_rw(iter), rc;
155 long map_len = 0;
156 struct blk_dax_ctl dax = {
Dan Williams7a9eb202016-06-03 18:06:47 -0700157 .addr = ERR_PTR(-EIO),
Dan Williamsb2e0d162016-01-15 16:55:59 -0800158 };
Jan Kara069c77b2016-05-11 11:58:51 +0200159 unsigned blkbits = inode->i_blkbits;
160 sector_t file_blks = (i_size_read(inode) + (1 << blkbits) - 1)
161 >> blkbits;
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800162
Dan Williamsb2e0d162016-01-15 16:55:59 -0800163 if (rw == READ)
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800164 end = min(end, i_size_read(inode));
165
166 while (pos < end) {
Ross Zwisler2765cfb2015-08-18 13:55:40 -0600167 size_t len;
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800168 if (pos == max) {
Jeff Moyere94f5a22015-08-14 16:15:31 -0400169 long page = pos >> PAGE_SHIFT;
170 sector_t block = page << (PAGE_SHIFT - blkbits);
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800171 unsigned first = pos - (block << blkbits);
172 long size;
173
174 if (pos == bh_max) {
175 bh->b_size = PAGE_ALIGN(end - pos);
176 bh->b_state = 0;
Dan Williamsb2e0d162016-01-15 16:55:59 -0800177 rc = get_block(inode, block, bh, rw == WRITE);
178 if (rc)
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800179 break;
180 if (!buffer_size_valid(bh))
181 bh->b_size = 1 << blkbits;
182 bh_max = pos - first + bh->b_size;
Dan Williamsb2e0d162016-01-15 16:55:59 -0800183 bdev = bh->b_bdev;
Jan Kara069c77b2016-05-11 11:58:51 +0200184 /*
185 * We allow uninitialized buffers for writes
186 * beyond EOF as those cannot race with faults
187 */
188 WARN_ON_ONCE(
189 (buffer_new(bh) && block < file_blks) ||
190 (rw == WRITE && buffer_unwritten(bh)));
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800191 } else {
192 unsigned done = bh->b_size -
193 (bh_max - (pos - first));
194 bh->b_blocknr += done >> blkbits;
195 bh->b_size -= done;
196 }
197
Dan Williamsb2e0d162016-01-15 16:55:59 -0800198 hole = rw == READ && !buffer_written(bh);
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800199 if (hole) {
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800200 size = bh->b_size - first;
201 } else {
Dan Williamsb2e0d162016-01-15 16:55:59 -0800202 dax_unmap_atomic(bdev, &dax);
203 dax.sector = to_sector(bh, inode);
204 dax.size = bh->b_size;
205 map_len = dax_map_atomic(bdev, &dax);
206 if (map_len < 0) {
207 rc = map_len;
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800208 break;
Dan Williamsb2e0d162016-01-15 16:55:59 -0800209 }
Dan Williamsb2e0d162016-01-15 16:55:59 -0800210 dax.addr += first;
211 size = map_len - first;
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800212 }
Eric Sandeen02395432016-06-23 16:54:46 -0500213 /*
214 * pos + size is one past the last offset for IO,
215 * so pos + size can overflow loff_t at extreme offsets.
216 * Cast to u64 to catch this and get the true minimum.
217 */
218 max = min_t(u64, pos + size, end);
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800219 }
220
Ross Zwisler2765cfb2015-08-18 13:55:40 -0600221 if (iov_iter_rw(iter) == WRITE) {
Dan Williamsb2e0d162016-01-15 16:55:59 -0800222 len = copy_from_iter_pmem(dax.addr, max - pos, iter);
Ross Zwisler2765cfb2015-08-18 13:55:40 -0600223 } else if (!hole)
Dan Williamsb2e0d162016-01-15 16:55:59 -0800224 len = copy_to_iter((void __force *) dax.addr, max - pos,
Ross Zwislere2e05392015-08-18 13:55:41 -0600225 iter);
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800226 else
227 len = iov_iter_zero(max - pos, iter);
228
Al Virocadfbb62015-11-10 19:42:49 -0700229 if (!len) {
Dan Williamsb2e0d162016-01-15 16:55:59 -0800230 rc = -EFAULT;
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800231 break;
Al Virocadfbb62015-11-10 19:42:49 -0700232 }
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800233
234 pos += len;
Dan Williamsb2e0d162016-01-15 16:55:59 -0800235 if (!IS_ERR(dax.addr))
236 dax.addr += len;
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800237 }
238
Dan Williamsb2e0d162016-01-15 16:55:59 -0800239 dax_unmap_atomic(bdev, &dax);
Ross Zwisler2765cfb2015-08-18 13:55:40 -0600240
Dan Williamsb2e0d162016-01-15 16:55:59 -0800241 return (pos == start) ? rc : pos - start;
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800242}
243
244/**
245 * dax_do_io - Perform I/O to a DAX file
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800246 * @iocb: The control block for this I/O
247 * @inode: The file which the I/O is directed at
248 * @iter: The addresses to do I/O from or to
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800249 * @get_block: The filesystem method used to translate file offsets to blocks
250 * @end_io: A filesystem callback for I/O completion
251 * @flags: See below
252 *
253 * This function uses the same locking scheme as do_blockdev_direct_IO:
254 * If @flags has DIO_LOCKING set, we assume that the i_mutex is held by the
255 * caller for writes. For reads, we take and release the i_mutex ourselves.
256 * If DIO_LOCKING is not set, the filesystem takes care of its own locking.
257 * As with do_blockdev_direct_IO(), we increment i_dio_count while the I/O
258 * is in progress.
259 */
Omar Sandovala95cd632015-03-16 04:33:51 -0700260ssize_t dax_do_io(struct kiocb *iocb, struct inode *inode,
Christoph Hellwigc8b8e322016-04-07 08:51:58 -0700261 struct iov_iter *iter, get_block_t get_block,
Omar Sandovala95cd632015-03-16 04:33:51 -0700262 dio_iodone_t end_io, int flags)
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800263{
264 struct buffer_head bh;
265 ssize_t retval = -EINVAL;
Christoph Hellwigc8b8e322016-04-07 08:51:58 -0700266 loff_t pos = iocb->ki_pos;
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800267 loff_t end = pos + iov_iter_count(iter);
268
269 memset(&bh, 0, sizeof(bh));
Ross Zwislereab95db2016-01-22 15:10:59 -0800270 bh.b_bdev = inode->i_sb->s_bdev;
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800271
Jan Karac3d98e32016-05-11 11:58:52 +0200272 if ((flags & DIO_LOCKING) && iov_iter_rw(iter) == READ)
Al Viro59551022016-01-22 15:40:57 -0500273 inode_lock(inode);
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800274
275 /* Protects against truncate */
Matthew Wilcoxbbab37d2015-07-03 10:40:42 -0400276 if (!(flags & DIO_SKIP_DIO_COUNT))
277 inode_dio_begin(inode);
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800278
Omar Sandovala95cd632015-03-16 04:33:51 -0700279 retval = dax_io(inode, iter, pos, end, get_block, &bh);
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800280
Omar Sandovala95cd632015-03-16 04:33:51 -0700281 if ((flags & DIO_LOCKING) && iov_iter_rw(iter) == READ)
Al Viro59551022016-01-22 15:40:57 -0500282 inode_unlock(inode);
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800283
Christoph Hellwig187372a2016-02-08 14:40:51 +1100284 if (end_io) {
285 int err;
286
287 err = end_io(iocb, pos, retval, bh.b_private);
288 if (err)
289 retval = err;
290 }
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800291
Matthew Wilcoxbbab37d2015-07-03 10:40:42 -0400292 if (!(flags & DIO_SKIP_DIO_COUNT))
293 inode_dio_end(inode);
Matthew Wilcoxd475c632015-02-16 15:58:56 -0800294 return retval;
295}
296EXPORT_SYMBOL_GPL(dax_do_io);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800297
298/*
Jan Karaac401cc2016-05-12 18:29:18 +0200299 * DAX radix tree locking
300 */
301struct exceptional_entry_key {
302 struct address_space *mapping;
303 unsigned long index;
304};
305
306struct wait_exceptional_entry_queue {
307 wait_queue_t wait;
308 struct exceptional_entry_key key;
309};
310
311static int wake_exceptional_entry_func(wait_queue_t *wait, unsigned int mode,
312 int sync, void *keyp)
313{
314 struct exceptional_entry_key *key = keyp;
315 struct wait_exceptional_entry_queue *ewait =
316 container_of(wait, struct wait_exceptional_entry_queue, wait);
317
318 if (key->mapping != ewait->key.mapping ||
319 key->index != ewait->key.index)
320 return 0;
321 return autoremove_wake_function(wait, mode, sync, NULL);
322}
323
324/*
325 * Check whether the given slot is locked. The function must be called with
326 * mapping->tree_lock held
327 */
328static inline int slot_locked(struct address_space *mapping, void **slot)
329{
330 unsigned long entry = (unsigned long)
331 radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
332 return entry & RADIX_DAX_ENTRY_LOCK;
333}
334
335/*
336 * Mark the given slot is locked. The function must be called with
337 * mapping->tree_lock held
338 */
339static inline void *lock_slot(struct address_space *mapping, void **slot)
340{
341 unsigned long entry = (unsigned long)
342 radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
343
344 entry |= RADIX_DAX_ENTRY_LOCK;
345 radix_tree_replace_slot(slot, (void *)entry);
346 return (void *)entry;
347}
348
349/*
350 * Mark the given slot is unlocked. The function must be called with
351 * mapping->tree_lock held
352 */
353static inline void *unlock_slot(struct address_space *mapping, void **slot)
354{
355 unsigned long entry = (unsigned long)
356 radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
357
358 entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK;
359 radix_tree_replace_slot(slot, (void *)entry);
360 return (void *)entry;
361}
362
363/*
364 * Lookup entry in radix tree, wait for it to become unlocked if it is
365 * exceptional entry and return it. The caller must call
366 * put_unlocked_mapping_entry() when he decided not to lock the entry or
367 * put_locked_mapping_entry() when he locked the entry and now wants to
368 * unlock it.
369 *
370 * The function must be called with mapping->tree_lock held.
371 */
372static void *get_unlocked_mapping_entry(struct address_space *mapping,
373 pgoff_t index, void ***slotp)
374{
375 void *ret, **slot;
376 struct wait_exceptional_entry_queue ewait;
377 wait_queue_head_t *wq = dax_entry_waitqueue(mapping, index);
378
379 init_wait(&ewait.wait);
380 ewait.wait.func = wake_exceptional_entry_func;
381 ewait.key.mapping = mapping;
382 ewait.key.index = index;
383
384 for (;;) {
385 ret = __radix_tree_lookup(&mapping->page_tree, index, NULL,
386 &slot);
387 if (!ret || !radix_tree_exceptional_entry(ret) ||
388 !slot_locked(mapping, slot)) {
389 if (slotp)
390 *slotp = slot;
391 return ret;
392 }
393 prepare_to_wait_exclusive(wq, &ewait.wait,
394 TASK_UNINTERRUPTIBLE);
395 spin_unlock_irq(&mapping->tree_lock);
396 schedule();
397 finish_wait(wq, &ewait.wait);
398 spin_lock_irq(&mapping->tree_lock);
399 }
400}
401
402/*
403 * Find radix tree entry at given index. If it points to a page, return with
404 * the page locked. If it points to the exceptional entry, return with the
405 * radix tree entry locked. If the radix tree doesn't contain given index,
406 * create empty exceptional entry for the index and return with it locked.
407 *
408 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
409 * persistent memory the benefit is doubtful. We can add that later if we can
410 * show it helps.
411 */
412static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index)
413{
414 void *ret, **slot;
415
416restart:
417 spin_lock_irq(&mapping->tree_lock);
418 ret = get_unlocked_mapping_entry(mapping, index, &slot);
419 /* No entry for given index? Make sure radix tree is big enough. */
420 if (!ret) {
421 int err;
422
423 spin_unlock_irq(&mapping->tree_lock);
424 err = radix_tree_preload(
425 mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM);
426 if (err)
427 return ERR_PTR(err);
428 ret = (void *)(RADIX_TREE_EXCEPTIONAL_ENTRY |
429 RADIX_DAX_ENTRY_LOCK);
430 spin_lock_irq(&mapping->tree_lock);
431 err = radix_tree_insert(&mapping->page_tree, index, ret);
432 radix_tree_preload_end();
433 if (err) {
434 spin_unlock_irq(&mapping->tree_lock);
435 /* Someone already created the entry? */
436 if (err == -EEXIST)
437 goto restart;
438 return ERR_PTR(err);
439 }
440 /* Good, we have inserted empty locked entry into the tree. */
441 mapping->nrexceptional++;
442 spin_unlock_irq(&mapping->tree_lock);
443 return ret;
444 }
445 /* Normal page in radix tree? */
446 if (!radix_tree_exceptional_entry(ret)) {
447 struct page *page = ret;
448
449 get_page(page);
450 spin_unlock_irq(&mapping->tree_lock);
451 lock_page(page);
452 /* Page got truncated? Retry... */
453 if (unlikely(page->mapping != mapping)) {
454 unlock_page(page);
455 put_page(page);
456 goto restart;
457 }
458 return page;
459 }
460 ret = lock_slot(mapping, slot);
461 spin_unlock_irq(&mapping->tree_lock);
462 return ret;
463}
464
465void dax_wake_mapping_entry_waiter(struct address_space *mapping,
466 pgoff_t index, bool wake_all)
467{
468 wait_queue_head_t *wq = dax_entry_waitqueue(mapping, index);
469
470 /*
471 * Checking for locked entry and prepare_to_wait_exclusive() happens
472 * under mapping->tree_lock, ditto for entry handling in our callers.
473 * So at this point all tasks that could have seen our entry locked
474 * must be in the waitqueue and the following check will see them.
475 */
476 if (waitqueue_active(wq)) {
477 struct exceptional_entry_key key;
478
479 key.mapping = mapping;
480 key.index = index;
481 __wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
482 }
483}
484
Jan Karabc2466e2016-05-12 18:29:19 +0200485void dax_unlock_mapping_entry(struct address_space *mapping, pgoff_t index)
Jan Karaac401cc2016-05-12 18:29:18 +0200486{
487 void *ret, **slot;
488
489 spin_lock_irq(&mapping->tree_lock);
490 ret = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot);
491 if (WARN_ON_ONCE(!ret || !radix_tree_exceptional_entry(ret) ||
492 !slot_locked(mapping, slot))) {
493 spin_unlock_irq(&mapping->tree_lock);
494 return;
495 }
496 unlock_slot(mapping, slot);
497 spin_unlock_irq(&mapping->tree_lock);
498 dax_wake_mapping_entry_waiter(mapping, index, false);
499}
500
501static void put_locked_mapping_entry(struct address_space *mapping,
502 pgoff_t index, void *entry)
503{
504 if (!radix_tree_exceptional_entry(entry)) {
505 unlock_page(entry);
506 put_page(entry);
507 } else {
Jan Karabc2466e2016-05-12 18:29:19 +0200508 dax_unlock_mapping_entry(mapping, index);
Jan Karaac401cc2016-05-12 18:29:18 +0200509 }
510}
511
512/*
513 * Called when we are done with radix tree entry we looked up via
514 * get_unlocked_mapping_entry() and which we didn't lock in the end.
515 */
516static void put_unlocked_mapping_entry(struct address_space *mapping,
517 pgoff_t index, void *entry)
518{
519 if (!radix_tree_exceptional_entry(entry))
520 return;
521
522 /* We have to wake up next waiter for the radix tree entry lock */
523 dax_wake_mapping_entry_waiter(mapping, index, false);
524}
525
526/*
527 * Delete exceptional DAX entry at @index from @mapping. Wait for radix tree
528 * entry to get unlocked before deleting it.
529 */
530int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
531{
532 void *entry;
533
534 spin_lock_irq(&mapping->tree_lock);
535 entry = get_unlocked_mapping_entry(mapping, index, NULL);
536 /*
537 * This gets called from truncate / punch_hole path. As such, the caller
538 * must hold locks protecting against concurrent modifications of the
539 * radix tree (usually fs-private i_mmap_sem for writing). Since the
540 * caller has seen exceptional entry for this index, we better find it
541 * at that index as well...
542 */
543 if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry))) {
544 spin_unlock_irq(&mapping->tree_lock);
545 return 0;
546 }
547 radix_tree_delete(&mapping->page_tree, index);
548 mapping->nrexceptional--;
549 spin_unlock_irq(&mapping->tree_lock);
550 dax_wake_mapping_entry_waiter(mapping, index, true);
551
552 return 1;
553}
554
555/*
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800556 * The user has performed a load from a hole in the file. Allocating
557 * a new page in the file would cause excessive storage usage for
558 * workloads with sparse files. We allocate a page cache page instead.
559 * We'll kick it out of the page cache if it's ever written to,
560 * otherwise it will simply fall out of the page cache under memory
561 * pressure without ever having been dirtied.
562 */
Jan Karaac401cc2016-05-12 18:29:18 +0200563static int dax_load_hole(struct address_space *mapping, void *entry,
564 struct vm_fault *vmf)
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800565{
Jan Karaac401cc2016-05-12 18:29:18 +0200566 struct page *page;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800567
Jan Karaac401cc2016-05-12 18:29:18 +0200568 /* Hole page already exists? Return it... */
569 if (!radix_tree_exceptional_entry(entry)) {
570 vmf->page = entry;
571 return VM_FAULT_LOCKED;
572 }
573
574 /* This will replace locked radix tree entry with a hole page */
575 page = find_or_create_page(mapping, vmf->pgoff,
576 vmf->gfp_mask | __GFP_ZERO);
577 if (!page) {
578 put_locked_mapping_entry(mapping, vmf->pgoff, entry);
579 return VM_FAULT_OOM;
580 }
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800581 vmf->page = page;
582 return VM_FAULT_LOCKED;
583}
584
Christoph Hellwigb0d5e822016-09-19 11:24:49 +1000585static int copy_user_dax(struct block_device *bdev, sector_t sector, size_t size,
586 struct page *to, unsigned long vaddr)
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800587{
Dan Williamsb2e0d162016-01-15 16:55:59 -0800588 struct blk_dax_ctl dax = {
Christoph Hellwigb0d5e822016-09-19 11:24:49 +1000589 .sector = sector,
590 .size = size,
Dan Williamsb2e0d162016-01-15 16:55:59 -0800591 };
Ross Zwislere2e05392015-08-18 13:55:41 -0600592 void *vto;
593
Dan Williamsb2e0d162016-01-15 16:55:59 -0800594 if (dax_map_atomic(bdev, &dax) < 0)
595 return PTR_ERR(dax.addr);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800596 vto = kmap_atomic(to);
Dan Williamsb2e0d162016-01-15 16:55:59 -0800597 copy_user_page(vto, (void __force *)dax.addr, vaddr, to);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800598 kunmap_atomic(vto);
Dan Williamsb2e0d162016-01-15 16:55:59 -0800599 dax_unmap_atomic(bdev, &dax);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800600 return 0;
601}
602
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300603#define DAX_PMD_INDEX(page_index) (page_index & (PMD_MASK >> PAGE_SHIFT))
Ross Zwisler9973c982016-01-22 15:10:47 -0800604
Jan Karaac401cc2016-05-12 18:29:18 +0200605static void *dax_insert_mapping_entry(struct address_space *mapping,
606 struct vm_fault *vmf,
607 void *entry, sector_t sector)
Ross Zwisler9973c982016-01-22 15:10:47 -0800608{
609 struct radix_tree_root *page_tree = &mapping->page_tree;
Jan Karaac401cc2016-05-12 18:29:18 +0200610 int error = 0;
611 bool hole_fill = false;
612 void *new_entry;
613 pgoff_t index = vmf->pgoff;
Ross Zwisler9973c982016-01-22 15:10:47 -0800614
Jan Karaac401cc2016-05-12 18:29:18 +0200615 if (vmf->flags & FAULT_FLAG_WRITE)
Dmitry Monakhovd2b2a282016-02-05 15:36:55 -0800616 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
Ross Zwisler9973c982016-01-22 15:10:47 -0800617
Jan Karaac401cc2016-05-12 18:29:18 +0200618 /* Replacing hole page with block mapping? */
619 if (!radix_tree_exceptional_entry(entry)) {
620 hole_fill = true;
621 /*
622 * Unmap the page now before we remove it from page cache below.
623 * The page is locked so it cannot be faulted in again.
624 */
625 unmap_mapping_range(mapping, vmf->pgoff << PAGE_SHIFT,
626 PAGE_SIZE, 0);
627 error = radix_tree_preload(vmf->gfp_mask & ~__GFP_HIGHMEM);
628 if (error)
629 return ERR_PTR(error);
Ross Zwisler9973c982016-01-22 15:10:47 -0800630 }
631
Jan Karaac401cc2016-05-12 18:29:18 +0200632 spin_lock_irq(&mapping->tree_lock);
633 new_entry = (void *)((unsigned long)RADIX_DAX_ENTRY(sector, false) |
634 RADIX_DAX_ENTRY_LOCK);
635 if (hole_fill) {
636 __delete_from_page_cache(entry, NULL);
637 /* Drop pagecache reference */
638 put_page(entry);
639 error = radix_tree_insert(page_tree, index, new_entry);
640 if (error) {
641 new_entry = ERR_PTR(error);
Ross Zwisler9973c982016-01-22 15:10:47 -0800642 goto unlock;
643 }
Jan Karaac401cc2016-05-12 18:29:18 +0200644 mapping->nrexceptional++;
645 } else {
646 void **slot;
647 void *ret;
Ross Zwisler9973c982016-01-22 15:10:47 -0800648
Jan Karaac401cc2016-05-12 18:29:18 +0200649 ret = __radix_tree_lookup(page_tree, index, NULL, &slot);
650 WARN_ON_ONCE(ret != entry);
651 radix_tree_replace_slot(slot, new_entry);
Ross Zwisler9973c982016-01-22 15:10:47 -0800652 }
Jan Karaac401cc2016-05-12 18:29:18 +0200653 if (vmf->flags & FAULT_FLAG_WRITE)
Ross Zwisler9973c982016-01-22 15:10:47 -0800654 radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY);
655 unlock:
656 spin_unlock_irq(&mapping->tree_lock);
Jan Karaac401cc2016-05-12 18:29:18 +0200657 if (hole_fill) {
658 radix_tree_preload_end();
659 /*
660 * We don't need hole page anymore, it has been replaced with
661 * locked radix tree entry now.
662 */
663 if (mapping->a_ops->freepage)
664 mapping->a_ops->freepage(entry);
665 unlock_page(entry);
666 put_page(entry);
667 }
668 return new_entry;
Ross Zwisler9973c982016-01-22 15:10:47 -0800669}
670
671static int dax_writeback_one(struct block_device *bdev,
672 struct address_space *mapping, pgoff_t index, void *entry)
673{
674 struct radix_tree_root *page_tree = &mapping->page_tree;
675 int type = RADIX_DAX_TYPE(entry);
676 struct radix_tree_node *node;
677 struct blk_dax_ctl dax;
678 void **slot;
679 int ret = 0;
680
681 spin_lock_irq(&mapping->tree_lock);
682 /*
683 * Regular page slots are stabilized by the page lock even
684 * without the tree itself locked. These unlocked entries
685 * need verification under the tree lock.
686 */
687 if (!__radix_tree_lookup(page_tree, index, &node, &slot))
688 goto unlock;
689 if (*slot != entry)
690 goto unlock;
691
692 /* another fsync thread may have already written back this entry */
693 if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))
694 goto unlock;
695
696 if (WARN_ON_ONCE(type != RADIX_DAX_PTE && type != RADIX_DAX_PMD)) {
697 ret = -EIO;
698 goto unlock;
699 }
700
701 dax.sector = RADIX_DAX_SECTOR(entry);
702 dax.size = (type == RADIX_DAX_PMD ? PMD_SIZE : PAGE_SIZE);
703 spin_unlock_irq(&mapping->tree_lock);
704
705 /*
706 * We cannot hold tree_lock while calling dax_map_atomic() because it
707 * eventually calls cond_resched().
708 */
709 ret = dax_map_atomic(bdev, &dax);
710 if (ret < 0)
711 return ret;
712
713 if (WARN_ON_ONCE(ret < dax.size)) {
714 ret = -EIO;
715 goto unmap;
716 }
717
718 wb_cache_pmem(dax.addr, dax.size);
719
720 spin_lock_irq(&mapping->tree_lock);
721 radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE);
722 spin_unlock_irq(&mapping->tree_lock);
723 unmap:
724 dax_unmap_atomic(bdev, &dax);
725 return ret;
726
727 unlock:
728 spin_unlock_irq(&mapping->tree_lock);
729 return ret;
730}
731
732/*
733 * Flush the mapping to the persistent domain within the byte range of [start,
734 * end]. This is required by data integrity operations to ensure file data is
735 * on persistent storage prior to completion of the operation.
736 */
Ross Zwisler7f6d5b52016-02-26 15:19:55 -0800737int dax_writeback_mapping_range(struct address_space *mapping,
738 struct block_device *bdev, struct writeback_control *wbc)
Ross Zwisler9973c982016-01-22 15:10:47 -0800739{
740 struct inode *inode = mapping->host;
Ross Zwisler9973c982016-01-22 15:10:47 -0800741 pgoff_t start_index, end_index, pmd_index;
742 pgoff_t indices[PAGEVEC_SIZE];
743 struct pagevec pvec;
744 bool done = false;
745 int i, ret = 0;
746 void *entry;
747
748 if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
749 return -EIO;
750
Ross Zwisler7f6d5b52016-02-26 15:19:55 -0800751 if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
752 return 0;
753
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300754 start_index = wbc->range_start >> PAGE_SHIFT;
755 end_index = wbc->range_end >> PAGE_SHIFT;
Ross Zwisler9973c982016-01-22 15:10:47 -0800756 pmd_index = DAX_PMD_INDEX(start_index);
757
758 rcu_read_lock();
759 entry = radix_tree_lookup(&mapping->page_tree, pmd_index);
760 rcu_read_unlock();
761
762 /* see if the start of our range is covered by a PMD entry */
763 if (entry && RADIX_DAX_TYPE(entry) == RADIX_DAX_PMD)
764 start_index = pmd_index;
765
766 tag_pages_for_writeback(mapping, start_index, end_index);
767
768 pagevec_init(&pvec, 0);
769 while (!done) {
770 pvec.nr = find_get_entries_tag(mapping, start_index,
771 PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE,
772 pvec.pages, indices);
773
774 if (pvec.nr == 0)
775 break;
776
777 for (i = 0; i < pvec.nr; i++) {
778 if (indices[i] > end_index) {
779 done = true;
780 break;
781 }
782
783 ret = dax_writeback_one(bdev, mapping, indices[i],
784 pvec.pages[i]);
785 if (ret < 0)
786 return ret;
787 }
788 }
Ross Zwisler9973c982016-01-22 15:10:47 -0800789 return 0;
790}
791EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);
792
Jan Karaac401cc2016-05-12 18:29:18 +0200793static int dax_insert_mapping(struct address_space *mapping,
Christoph Hellwig1aaba092016-09-19 11:24:49 +1000794 struct block_device *bdev, sector_t sector, size_t size,
795 void **entryp, struct vm_area_struct *vma, struct vm_fault *vmf)
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800796{
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800797 unsigned long vaddr = (unsigned long)vmf->virtual_address;
Dan Williamsb2e0d162016-01-15 16:55:59 -0800798 struct blk_dax_ctl dax = {
Christoph Hellwig1aaba092016-09-19 11:24:49 +1000799 .sector = sector,
800 .size = size,
Dan Williamsb2e0d162016-01-15 16:55:59 -0800801 };
Jan Karaac401cc2016-05-12 18:29:18 +0200802 void *ret;
803 void *entry = *entryp;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800804
Jan Kara4d9a2c82016-05-12 18:29:20 +0200805 if (dax_map_atomic(bdev, &dax) < 0)
806 return PTR_ERR(dax.addr);
Dan Williamsb2e0d162016-01-15 16:55:59 -0800807 dax_unmap_atomic(bdev, &dax);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800808
Jan Karaac401cc2016-05-12 18:29:18 +0200809 ret = dax_insert_mapping_entry(mapping, vmf, entry, dax.sector);
Jan Kara4d9a2c82016-05-12 18:29:20 +0200810 if (IS_ERR(ret))
811 return PTR_ERR(ret);
Jan Karaac401cc2016-05-12 18:29:18 +0200812 *entryp = ret;
Ross Zwisler9973c982016-01-22 15:10:47 -0800813
Jan Kara4d9a2c82016-05-12 18:29:20 +0200814 return vm_insert_mixed(vma, vaddr, dax.pfn);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800815}
816
Dave Chinnerce5c5d52015-06-04 09:18:18 +1000817/**
Ross Zwisler6b524992016-07-26 15:21:05 -0700818 * dax_fault - handle a page fault on a DAX file
Dave Chinnerce5c5d52015-06-04 09:18:18 +1000819 * @vma: The virtual memory area where the fault occurred
820 * @vmf: The description of the fault
821 * @get_block: The filesystem method used to translate file offsets to blocks
822 *
823 * When a page fault occurs, filesystems may call this helper in their
Ross Zwisler6b524992016-07-26 15:21:05 -0700824 * fault handler for DAX files. dax_fault() assumes the caller has done all
Dave Chinnerce5c5d52015-06-04 09:18:18 +1000825 * the necessary locking for the page fault to proceed successfully.
826 */
Ross Zwisler6b524992016-07-26 15:21:05 -0700827int dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
Jan Kara02fbd132016-05-11 11:58:48 +0200828 get_block_t get_block)
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800829{
830 struct file *file = vma->vm_file;
831 struct address_space *mapping = file->f_mapping;
832 struct inode *inode = mapping->host;
Jan Karaac401cc2016-05-12 18:29:18 +0200833 void *entry;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800834 struct buffer_head bh;
835 unsigned long vaddr = (unsigned long)vmf->virtual_address;
836 unsigned blkbits = inode->i_blkbits;
837 sector_t block;
838 pgoff_t size;
839 int error;
840 int major = 0;
841
Jan Karaac401cc2016-05-12 18:29:18 +0200842 /*
843 * Check whether offset isn't beyond end of file now. Caller is supposed
844 * to hold locks serializing us with truncate / punch hole so this is
845 * a reliable test.
846 */
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800847 size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
848 if (vmf->pgoff >= size)
849 return VM_FAULT_SIGBUS;
850
851 memset(&bh, 0, sizeof(bh));
852 block = (sector_t)vmf->pgoff << (PAGE_SHIFT - blkbits);
Ross Zwislereab95db2016-01-22 15:10:59 -0800853 bh.b_bdev = inode->i_sb->s_bdev;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800854 bh.b_size = PAGE_SIZE;
855
Jan Karaac401cc2016-05-12 18:29:18 +0200856 entry = grab_mapping_entry(mapping, vmf->pgoff);
857 if (IS_ERR(entry)) {
858 error = PTR_ERR(entry);
859 goto out;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800860 }
861
862 error = get_block(inode, block, &bh, 0);
863 if (!error && (bh.b_size < PAGE_SIZE))
864 error = -EIO; /* fs corruption? */
865 if (error)
Jan Karaac401cc2016-05-12 18:29:18 +0200866 goto unlock_entry;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800867
868 if (vmf->cow_page) {
869 struct page *new_page = vmf->cow_page;
870 if (buffer_written(&bh))
Christoph Hellwigb0d5e822016-09-19 11:24:49 +1000871 error = copy_user_dax(bh.b_bdev, to_sector(&bh, inode),
872 bh.b_size, new_page, vaddr);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800873 else
874 clear_user_highpage(new_page, vaddr);
875 if (error)
Jan Karaac401cc2016-05-12 18:29:18 +0200876 goto unlock_entry;
877 if (!radix_tree_exceptional_entry(entry)) {
878 vmf->page = entry;
Jan Karabc2466e2016-05-12 18:29:19 +0200879 return VM_FAULT_LOCKED;
Jan Karaac401cc2016-05-12 18:29:18 +0200880 }
Jan Karabc2466e2016-05-12 18:29:19 +0200881 vmf->entry = entry;
882 return VM_FAULT_DAX_LOCKED;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800883 }
884
Jan Karaac401cc2016-05-12 18:29:18 +0200885 if (!buffer_mapped(&bh)) {
886 if (vmf->flags & FAULT_FLAG_WRITE) {
887 error = get_block(inode, block, &bh, 1);
888 count_vm_event(PGMAJFAULT);
889 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
890 major = VM_FAULT_MAJOR;
891 if (!error && (bh.b_size < PAGE_SIZE))
892 error = -EIO;
893 if (error)
894 goto unlock_entry;
895 } else {
896 return dax_load_hole(mapping, entry, vmf);
897 }
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800898 }
899
Jan Kara02fbd132016-05-11 11:58:48 +0200900 /* Filesystem should not return unwritten buffers to us! */
Jan Kara2b109452016-05-11 11:58:50 +0200901 WARN_ON_ONCE(buffer_unwritten(&bh) || buffer_new(&bh));
Christoph Hellwig1aaba092016-09-19 11:24:49 +1000902 error = dax_insert_mapping(mapping, bh.b_bdev, to_sector(&bh, inode),
903 bh.b_size, &entry, vma, vmf);
Jan Karaac401cc2016-05-12 18:29:18 +0200904 unlock_entry:
905 put_locked_mapping_entry(mapping, vmf->pgoff, entry);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800906 out:
907 if (error == -ENOMEM)
908 return VM_FAULT_OOM | major;
909 /* -EBUSY is fine, somebody else faulted on the same PTE */
910 if ((error < 0) && (error != -EBUSY))
911 return VM_FAULT_SIGBUS | major;
912 return VM_FAULT_NOPAGE | major;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800913}
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800914EXPORT_SYMBOL_GPL(dax_fault);
Matthew Wilcox4c0ccfe2015-02-16 15:59:06 -0800915
Jan Kara348e9672016-05-12 18:29:15 +0200916#if defined(CONFIG_TRANSPARENT_HUGEPAGE)
Matthew Wilcox844f35d2015-09-08 14:58:57 -0700917/*
918 * The 'colour' (ie low bits) within a PMD of a page offset. This comes up
919 * more often than one might expect in the below function.
920 */
921#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1)
922
Dan Williamscbb38e42016-01-15 16:56:58 -0800923static void __dax_dbg(struct buffer_head *bh, unsigned long address,
924 const char *reason, const char *fn)
925{
926 if (bh) {
927 char bname[BDEVNAME_SIZE];
928 bdevname(bh->b_bdev, bname);
929 pr_debug("%s: %s addr: %lx dev %s state %lx start %lld "
930 "length %zd fallback: %s\n", fn, current->comm,
931 address, bname, bh->b_state, (u64)bh->b_blocknr,
932 bh->b_size, reason);
933 } else {
934 pr_debug("%s: %s addr: %lx fallback: %s\n", fn,
935 current->comm, address, reason);
936 }
937}
938
939#define dax_pmd_dbg(bh, address, reason) __dax_dbg(bh, address, reason, "dax_pmd")
940
Ross Zwisler6b524992016-07-26 15:21:05 -0700941/**
942 * dax_pmd_fault - handle a PMD fault on a DAX file
943 * @vma: The virtual memory area where the fault occurred
944 * @vmf: The description of the fault
945 * @get_block: The filesystem method used to translate file offsets to blocks
946 *
947 * When a page fault occurs, filesystems may call this helper in their
948 * pmd_fault handler for DAX files.
949 */
950int dax_pmd_fault(struct vm_area_struct *vma, unsigned long address,
Jan Kara02fbd132016-05-11 11:58:48 +0200951 pmd_t *pmd, unsigned int flags, get_block_t get_block)
Matthew Wilcox844f35d2015-09-08 14:58:57 -0700952{
953 struct file *file = vma->vm_file;
954 struct address_space *mapping = file->f_mapping;
955 struct inode *inode = mapping->host;
956 struct buffer_head bh;
957 unsigned blkbits = inode->i_blkbits;
958 unsigned long pmd_addr = address & PMD_MASK;
959 bool write = flags & FAULT_FLAG_WRITE;
Dan Williamsb2e0d162016-01-15 16:55:59 -0800960 struct block_device *bdev;
Matthew Wilcox844f35d2015-09-08 14:58:57 -0700961 pgoff_t size, pgoff;
Dan Williamsb2e0d162016-01-15 16:55:59 -0800962 sector_t block;
Jan Karaac401cc2016-05-12 18:29:18 +0200963 int result = 0;
Ross Zwisler9973c982016-01-22 15:10:47 -0800964 bool alloc = false;
Matthew Wilcox844f35d2015-09-08 14:58:57 -0700965
Dan Williamsc046c322016-01-15 16:57:01 -0800966 /* dax pmd mappings require pfn_t_devmap() */
Dan Williamsee82c9e2015-11-15 16:06:32 -0800967 if (!IS_ENABLED(CONFIG_FS_DAX_PMD))
968 return VM_FAULT_FALLBACK;
969
Matthew Wilcox844f35d2015-09-08 14:58:57 -0700970 /* Fall back to PTEs if we're going to COW */
Toshi Kani59bf4fb2016-01-15 16:56:05 -0800971 if (write && !(vma->vm_flags & VM_SHARED)) {
972 split_huge_pmd(vma, pmd, address);
Dan Williamscbb38e42016-01-15 16:56:58 -0800973 dax_pmd_dbg(NULL, address, "cow write");
Matthew Wilcox844f35d2015-09-08 14:58:57 -0700974 return VM_FAULT_FALLBACK;
Toshi Kani59bf4fb2016-01-15 16:56:05 -0800975 }
Matthew Wilcox844f35d2015-09-08 14:58:57 -0700976 /* If the PMD would extend outside the VMA */
Dan Williamscbb38e42016-01-15 16:56:58 -0800977 if (pmd_addr < vma->vm_start) {
978 dax_pmd_dbg(NULL, address, "vma start unaligned");
Matthew Wilcox844f35d2015-09-08 14:58:57 -0700979 return VM_FAULT_FALLBACK;
Dan Williamscbb38e42016-01-15 16:56:58 -0800980 }
981 if ((pmd_addr + PMD_SIZE) > vma->vm_end) {
982 dax_pmd_dbg(NULL, address, "vma end unaligned");
Matthew Wilcox844f35d2015-09-08 14:58:57 -0700983 return VM_FAULT_FALLBACK;
Dan Williamscbb38e42016-01-15 16:56:58 -0800984 }
Matthew Wilcox844f35d2015-09-08 14:58:57 -0700985
Matthew Wilcox3fdd1b472015-09-08 14:59:39 -0700986 pgoff = linear_page_index(vma, pmd_addr);
Matthew Wilcox844f35d2015-09-08 14:58:57 -0700987 size = (i_size_read(inode) + PAGE_SIZE - 1) >> PAGE_SHIFT;
988 if (pgoff >= size)
989 return VM_FAULT_SIGBUS;
990 /* If the PMD would cover blocks out of the file */
Dan Williamscbb38e42016-01-15 16:56:58 -0800991 if ((pgoff | PG_PMD_COLOUR) >= size) {
992 dax_pmd_dbg(NULL, address,
993 "offset + huge page size > file size");
Matthew Wilcox844f35d2015-09-08 14:58:57 -0700994 return VM_FAULT_FALLBACK;
Dan Williamscbb38e42016-01-15 16:56:58 -0800995 }
Matthew Wilcox844f35d2015-09-08 14:58:57 -0700996
997 memset(&bh, 0, sizeof(bh));
Ross Zwislerd4bbe702016-01-22 15:10:31 -0800998 bh.b_bdev = inode->i_sb->s_bdev;
Matthew Wilcox844f35d2015-09-08 14:58:57 -0700999 block = (sector_t)pgoff << (PAGE_SHIFT - blkbits);
1000
1001 bh.b_size = PMD_SIZE;
Ross Zwisler9973c982016-01-22 15:10:47 -08001002
1003 if (get_block(inode, block, &bh, 0) != 0)
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001004 return VM_FAULT_SIGBUS;
Ross Zwisler9973c982016-01-22 15:10:47 -08001005
1006 if (!buffer_mapped(&bh) && write) {
1007 if (get_block(inode, block, &bh, 1) != 0)
1008 return VM_FAULT_SIGBUS;
1009 alloc = true;
Jan Kara2b109452016-05-11 11:58:50 +02001010 WARN_ON_ONCE(buffer_unwritten(&bh) || buffer_new(&bh));
Ross Zwisler9973c982016-01-22 15:10:47 -08001011 }
1012
Dan Williamsb2e0d162016-01-15 16:55:59 -08001013 bdev = bh.b_bdev;
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001014
1015 /*
1016 * If the filesystem isn't willing to tell us the length of a hole,
1017 * just fall back to PTEs. Calling get_block 512 times in a loop
1018 * would be silly.
1019 */
Dan Williamscbb38e42016-01-15 16:56:58 -08001020 if (!buffer_size_valid(&bh) || bh.b_size < PMD_SIZE) {
1021 dax_pmd_dbg(&bh, address, "allocated block too small");
Ross Zwisler9973c982016-01-22 15:10:47 -08001022 return VM_FAULT_FALLBACK;
Dan Williamscbb38e42016-01-15 16:56:58 -08001023 }
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001024
Ross Zwisler9973c982016-01-22 15:10:47 -08001025 /*
1026 * If we allocated new storage, make sure no process has any
1027 * zero pages covering this hole
1028 */
1029 if (alloc) {
1030 loff_t lstart = pgoff << PAGE_SHIFT;
1031 loff_t lend = lstart + PMD_SIZE - 1; /* inclusive */
1032
1033 truncate_pagecache_range(inode, lstart, lend);
1034 }
1035
Jan Karab9953532016-05-12 18:29:14 +02001036 if (!write && !buffer_mapped(&bh)) {
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001037 spinlock_t *ptl;
Kirill A. Shutemovd295e342015-09-08 14:59:34 -07001038 pmd_t entry;
Aaron Lu6fcb52a2016-10-07 17:00:08 -07001039 struct page *zero_page = mm_get_huge_zero_page(vma->vm_mm);
Kirill A. Shutemovd295e342015-09-08 14:59:34 -07001040
Dan Williamscbb38e42016-01-15 16:56:58 -08001041 if (unlikely(!zero_page)) {
1042 dax_pmd_dbg(&bh, address, "no zero page");
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001043 goto fallback;
Dan Williamscbb38e42016-01-15 16:56:58 -08001044 }
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001045
Kirill A. Shutemovd295e342015-09-08 14:59:34 -07001046 ptl = pmd_lock(vma->vm_mm, pmd);
1047 if (!pmd_none(*pmd)) {
1048 spin_unlock(ptl);
Dan Williamscbb38e42016-01-15 16:56:58 -08001049 dax_pmd_dbg(&bh, address, "pmd already present");
Kirill A. Shutemovd295e342015-09-08 14:59:34 -07001050 goto fallback;
1051 }
1052
Dan Williamscbb38e42016-01-15 16:56:58 -08001053 dev_dbg(part_to_dev(bdev->bd_part),
1054 "%s: %s addr: %lx pfn: <zero> sect: %llx\n",
1055 __func__, current->comm, address,
1056 (unsigned long long) to_sector(&bh, inode));
1057
Kirill A. Shutemovd295e342015-09-08 14:59:34 -07001058 entry = mk_pmd(zero_page, vma->vm_page_prot);
1059 entry = pmd_mkhuge(entry);
1060 set_pmd_at(vma->vm_mm, pmd_addr, pmd, entry);
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001061 result = VM_FAULT_NOPAGE;
Kirill A. Shutemovd295e342015-09-08 14:59:34 -07001062 spin_unlock(ptl);
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001063 } else {
Dan Williamsb2e0d162016-01-15 16:55:59 -08001064 struct blk_dax_ctl dax = {
1065 .sector = to_sector(&bh, inode),
1066 .size = PMD_SIZE,
1067 };
1068 long length = dax_map_atomic(bdev, &dax);
1069
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001070 if (length < 0) {
Dan Williams8b3db9792016-02-24 14:02:06 -08001071 dax_pmd_dbg(&bh, address, "dax-error fallback");
1072 goto fallback;
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001073 }
Dan Williamscbb38e42016-01-15 16:56:58 -08001074 if (length < PMD_SIZE) {
1075 dax_pmd_dbg(&bh, address, "dax-length too small");
1076 dax_unmap_atomic(bdev, &dax);
1077 goto fallback;
1078 }
1079 if (pfn_t_to_pfn(dax.pfn) & PG_PMD_COLOUR) {
1080 dax_pmd_dbg(&bh, address, "pfn unaligned");
Dan Williamsb2e0d162016-01-15 16:55:59 -08001081 dax_unmap_atomic(bdev, &dax);
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001082 goto fallback;
Dan Williamsb2e0d162016-01-15 16:55:59 -08001083 }
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001084
Dan Williamsc046c322016-01-15 16:57:01 -08001085 if (!pfn_t_devmap(dax.pfn)) {
Dan Williamsb2e0d162016-01-15 16:55:59 -08001086 dax_unmap_atomic(bdev, &dax);
Dan Williamscbb38e42016-01-15 16:56:58 -08001087 dax_pmd_dbg(&bh, address, "pfn not in memmap");
Dan Williams152d7bd2015-11-12 18:33:54 -08001088 goto fallback;
Dan Williamsb2e0d162016-01-15 16:55:59 -08001089 }
Dan Williamsb2e0d162016-01-15 16:55:59 -08001090 dax_unmap_atomic(bdev, &dax);
Ross Zwisler0f90cc62015-10-15 15:28:32 -07001091
Ross Zwisler9973c982016-01-22 15:10:47 -08001092 /*
1093 * For PTE faults we insert a radix tree entry for reads, and
1094 * leave it clean. Then on the first write we dirty the radix
1095 * tree entry via the dax_pfn_mkwrite() path. This sequence
1096 * allows the dax_pfn_mkwrite() call to be simpler and avoid a
1097 * call into get_block() to translate the pgoff to a sector in
1098 * order to be able to create a new radix tree entry.
1099 *
1100 * The PMD path doesn't have an equivalent to
1101 * dax_pfn_mkwrite(), though, so for a read followed by a
Ross Zwisler6b524992016-07-26 15:21:05 -07001102 * write we traverse all the way through dax_pmd_fault()
Ross Zwisler9973c982016-01-22 15:10:47 -08001103 * twice. This means we can just skip inserting a radix tree
1104 * entry completely on the initial read and just wait until
1105 * the write to insert a dirty entry.
1106 */
1107 if (write) {
Jan Karaac401cc2016-05-12 18:29:18 +02001108 /*
1109 * We should insert radix-tree entry and dirty it here.
1110 * For now this is broken...
1111 */
Ross Zwisler9973c982016-01-22 15:10:47 -08001112 }
1113
Dan Williamscbb38e42016-01-15 16:56:58 -08001114 dev_dbg(part_to_dev(bdev->bd_part),
1115 "%s: %s addr: %lx pfn: %lx sect: %llx\n",
1116 __func__, current->comm, address,
1117 pfn_t_to_pfn(dax.pfn),
1118 (unsigned long long) dax.sector);
Dan Williams34c0fd52016-01-15 16:56:14 -08001119 result |= vmf_insert_pfn_pmd(vma, address, pmd,
Dan Williamsf25748e32016-01-15 16:56:43 -08001120 dax.pfn, write);
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001121 }
1122
1123 out:
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001124 return result;
1125
1126 fallback:
1127 count_vm_event(THP_FAULT_FALLBACK);
1128 result = VM_FAULT_FALLBACK;
1129 goto out;
1130}
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001131EXPORT_SYMBOL_GPL(dax_pmd_fault);
Valentin Rothbergdd8a2b62015-09-08 14:59:09 -07001132#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
Matthew Wilcox844f35d2015-09-08 14:58:57 -07001133
Matthew Wilcox4c0ccfe2015-02-16 15:59:06 -08001134/**
Boaz Harrosh0e3b2102015-04-15 16:15:14 -07001135 * dax_pfn_mkwrite - handle first write to DAX page
1136 * @vma: The virtual memory area where the fault occurred
1137 * @vmf: The description of the fault
Boaz Harrosh0e3b2102015-04-15 16:15:14 -07001138 */
1139int dax_pfn_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
1140{
Ross Zwisler9973c982016-01-22 15:10:47 -08001141 struct file *file = vma->vm_file;
Jan Karaac401cc2016-05-12 18:29:18 +02001142 struct address_space *mapping = file->f_mapping;
1143 void *entry;
1144 pgoff_t index = vmf->pgoff;
Boaz Harrosh0e3b2102015-04-15 16:15:14 -07001145
Jan Karaac401cc2016-05-12 18:29:18 +02001146 spin_lock_irq(&mapping->tree_lock);
1147 entry = get_unlocked_mapping_entry(mapping, index, NULL);
1148 if (!entry || !radix_tree_exceptional_entry(entry))
1149 goto out;
1150 radix_tree_tag_set(&mapping->page_tree, index, PAGECACHE_TAG_DIRTY);
1151 put_unlocked_mapping_entry(mapping, index, entry);
1152out:
1153 spin_unlock_irq(&mapping->tree_lock);
Boaz Harrosh0e3b2102015-04-15 16:15:14 -07001154 return VM_FAULT_NOPAGE;
1155}
1156EXPORT_SYMBOL_GPL(dax_pfn_mkwrite);
1157
Vishal Verma4b0228f2016-04-21 15:13:46 -04001158static bool dax_range_is_aligned(struct block_device *bdev,
1159 unsigned int offset, unsigned int length)
1160{
1161 unsigned short sector_size = bdev_logical_block_size(bdev);
1162
1163 if (!IS_ALIGNED(offset, sector_size))
1164 return false;
1165 if (!IS_ALIGNED(length, sector_size))
1166 return false;
1167
1168 return true;
1169}
1170
Christoph Hellwig679c8bd2016-05-09 10:47:04 +02001171int __dax_zero_page_range(struct block_device *bdev, sector_t sector,
1172 unsigned int offset, unsigned int length)
1173{
1174 struct blk_dax_ctl dax = {
1175 .sector = sector,
1176 .size = PAGE_SIZE,
1177 };
1178
Vishal Verma4b0228f2016-04-21 15:13:46 -04001179 if (dax_range_is_aligned(bdev, offset, length)) {
1180 sector_t start_sector = dax.sector + (offset >> 9);
1181
1182 return blkdev_issue_zeroout(bdev, start_sector,
1183 length >> 9, GFP_NOFS, true);
1184 } else {
1185 if (dax_map_atomic(bdev, &dax) < 0)
1186 return PTR_ERR(dax.addr);
1187 clear_pmem(dax.addr + offset, length);
Vishal Verma4b0228f2016-04-21 15:13:46 -04001188 dax_unmap_atomic(bdev, &dax);
1189 }
Christoph Hellwig679c8bd2016-05-09 10:47:04 +02001190 return 0;
1191}
1192EXPORT_SYMBOL_GPL(__dax_zero_page_range);
1193
Boaz Harrosh0e3b2102015-04-15 16:15:14 -07001194/**
Matthew Wilcox25726bc2015-02-16 15:59:35 -08001195 * dax_zero_page_range - zero a range within a page of a DAX file
Matthew Wilcox4c0ccfe2015-02-16 15:59:06 -08001196 * @inode: The file being truncated
1197 * @from: The file offset that is being truncated to
Matthew Wilcox25726bc2015-02-16 15:59:35 -08001198 * @length: The number of bytes to zero
Matthew Wilcox4c0ccfe2015-02-16 15:59:06 -08001199 * @get_block: The filesystem method used to translate file offsets to blocks
1200 *
Matthew Wilcox25726bc2015-02-16 15:59:35 -08001201 * This function can be called by a filesystem when it is zeroing part of a
1202 * page in a DAX file. This is intended for hole-punch operations. If
1203 * you are truncating a file, the helper function dax_truncate_page() may be
1204 * more convenient.
Matthew Wilcox4c0ccfe2015-02-16 15:59:06 -08001205 */
Matthew Wilcox25726bc2015-02-16 15:59:35 -08001206int dax_zero_page_range(struct inode *inode, loff_t from, unsigned length,
1207 get_block_t get_block)
Matthew Wilcox4c0ccfe2015-02-16 15:59:06 -08001208{
1209 struct buffer_head bh;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001210 pgoff_t index = from >> PAGE_SHIFT;
1211 unsigned offset = from & (PAGE_SIZE-1);
Matthew Wilcox4c0ccfe2015-02-16 15:59:06 -08001212 int err;
1213
1214 /* Block boundary? Nothing to do */
1215 if (!length)
1216 return 0;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001217 BUG_ON((offset + length) > PAGE_SIZE);
Matthew Wilcox4c0ccfe2015-02-16 15:59:06 -08001218
1219 memset(&bh, 0, sizeof(bh));
Ross Zwislereab95db2016-01-22 15:10:59 -08001220 bh.b_bdev = inode->i_sb->s_bdev;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001221 bh.b_size = PAGE_SIZE;
Matthew Wilcox4c0ccfe2015-02-16 15:59:06 -08001222 err = get_block(inode, index, &bh, 0);
Christoph Hellwig679c8bd2016-05-09 10:47:04 +02001223 if (err < 0 || !buffer_written(&bh))
Matthew Wilcox4c0ccfe2015-02-16 15:59:06 -08001224 return err;
Dan Williamsb2e0d162016-01-15 16:55:59 -08001225
Christoph Hellwig679c8bd2016-05-09 10:47:04 +02001226 return __dax_zero_page_range(bh.b_bdev, to_sector(&bh, inode),
1227 offset, length);
Matthew Wilcox4c0ccfe2015-02-16 15:59:06 -08001228}
Matthew Wilcox25726bc2015-02-16 15:59:35 -08001229EXPORT_SYMBOL_GPL(dax_zero_page_range);
1230
1231/**
1232 * dax_truncate_page - handle a partial page being truncated in a DAX file
1233 * @inode: The file being truncated
1234 * @from: The file offset that is being truncated to
1235 * @get_block: The filesystem method used to translate file offsets to blocks
1236 *
1237 * Similar to block_truncate_page(), this function can be called by a
1238 * filesystem when it is truncating a DAX file to handle the partial page.
Matthew Wilcox25726bc2015-02-16 15:59:35 -08001239 */
1240int dax_truncate_page(struct inode *inode, loff_t from, get_block_t get_block)
1241{
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001242 unsigned length = PAGE_ALIGN(from) - from;
Matthew Wilcox25726bc2015-02-16 15:59:35 -08001243 return dax_zero_page_range(inode, from, length, get_block);
1244}
Matthew Wilcox4c0ccfe2015-02-16 15:59:06 -08001245EXPORT_SYMBOL_GPL(dax_truncate_page);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001246
1247#ifdef CONFIG_FS_IOMAP
1248static loff_t
1249iomap_dax_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
1250 struct iomap *iomap)
1251{
1252 struct iov_iter *iter = data;
1253 loff_t end = pos + length, done = 0;
1254 ssize_t ret = 0;
1255
1256 if (iov_iter_rw(iter) == READ) {
1257 end = min(end, i_size_read(inode));
1258 if (pos >= end)
1259 return 0;
1260
1261 if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
1262 return iov_iter_zero(min(length, end - pos), iter);
1263 }
1264
1265 if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))
1266 return -EIO;
1267
1268 while (pos < end) {
1269 unsigned offset = pos & (PAGE_SIZE - 1);
1270 struct blk_dax_ctl dax = { 0 };
1271 ssize_t map_len;
1272
1273 dax.sector = iomap->blkno +
1274 (((pos & PAGE_MASK) - iomap->offset) >> 9);
1275 dax.size = (length + offset + PAGE_SIZE - 1) & PAGE_MASK;
1276 map_len = dax_map_atomic(iomap->bdev, &dax);
1277 if (map_len < 0) {
1278 ret = map_len;
1279 break;
1280 }
1281
1282 dax.addr += offset;
1283 map_len -= offset;
1284 if (map_len > end - pos)
1285 map_len = end - pos;
1286
1287 if (iov_iter_rw(iter) == WRITE)
1288 map_len = copy_from_iter_pmem(dax.addr, map_len, iter);
1289 else
1290 map_len = copy_to_iter(dax.addr, map_len, iter);
1291 dax_unmap_atomic(iomap->bdev, &dax);
1292 if (map_len <= 0) {
1293 ret = map_len ? map_len : -EFAULT;
1294 break;
1295 }
1296
1297 pos += map_len;
1298 length -= map_len;
1299 done += map_len;
1300 }
1301
1302 return done ? done : ret;
1303}
1304
1305/**
1306 * iomap_dax_rw - Perform I/O to a DAX file
1307 * @iocb: The control block for this I/O
1308 * @iter: The addresses to do I/O from or to
1309 * @ops: iomap ops passed from the file system
1310 *
1311 * This function performs read and write operations to directly mapped
1312 * persistent memory. The callers needs to take care of read/write exclusion
1313 * and evicting any page cache pages in the region under I/O.
1314 */
1315ssize_t
1316iomap_dax_rw(struct kiocb *iocb, struct iov_iter *iter,
1317 struct iomap_ops *ops)
1318{
1319 struct address_space *mapping = iocb->ki_filp->f_mapping;
1320 struct inode *inode = mapping->host;
1321 loff_t pos = iocb->ki_pos, ret = 0, done = 0;
1322 unsigned flags = 0;
1323
1324 if (iov_iter_rw(iter) == WRITE)
1325 flags |= IOMAP_WRITE;
1326
1327 /*
1328 * Yes, even DAX files can have page cache attached to them: A zeroed
1329 * page is inserted into the pagecache when we have to serve a write
1330 * fault on a hole. It should never be dirtied and can simply be
1331 * dropped from the pagecache once we get real data for the page.
1332 *
1333 * XXX: This is racy against mmap, and there's nothing we can do about
1334 * it. We'll eventually need to shift this down even further so that
1335 * we can check if we allocated blocks over a hole first.
1336 */
1337 if (mapping->nrpages) {
1338 ret = invalidate_inode_pages2_range(mapping,
1339 pos >> PAGE_SHIFT,
1340 (pos + iov_iter_count(iter) - 1) >> PAGE_SHIFT);
1341 WARN_ON_ONCE(ret);
1342 }
1343
1344 while (iov_iter_count(iter)) {
1345 ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
1346 iter, iomap_dax_actor);
1347 if (ret <= 0)
1348 break;
1349 pos += ret;
1350 done += ret;
1351 }
1352
1353 iocb->ki_pos += done;
1354 return done ? done : ret;
1355}
1356EXPORT_SYMBOL_GPL(iomap_dax_rw);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001357
1358/**
1359 * iomap_dax_fault - handle a page fault on a DAX file
1360 * @vma: The virtual memory area where the fault occurred
1361 * @vmf: The description of the fault
1362 * @ops: iomap ops passed from the file system
1363 *
1364 * When a page fault occurs, filesystems may call this helper in their fault
1365 * or mkwrite handler for DAX files. Assumes the caller has done all the
1366 * necessary locking for the page fault to proceed successfully.
1367 */
1368int iomap_dax_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
1369 struct iomap_ops *ops)
1370{
1371 struct address_space *mapping = vma->vm_file->f_mapping;
1372 struct inode *inode = mapping->host;
1373 unsigned long vaddr = (unsigned long)vmf->virtual_address;
1374 loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
1375 sector_t sector;
1376 struct iomap iomap = { 0 };
1377 unsigned flags = 0;
1378 int error, major = 0;
1379 void *entry;
1380
1381 /*
1382 * Check whether offset isn't beyond end of file now. Caller is supposed
1383 * to hold locks serializing us with truncate / punch hole so this is
1384 * a reliable test.
1385 */
1386 if (pos >= i_size_read(inode))
1387 return VM_FAULT_SIGBUS;
1388
1389 entry = grab_mapping_entry(mapping, vmf->pgoff);
1390 if (IS_ERR(entry)) {
1391 error = PTR_ERR(entry);
1392 goto out;
1393 }
1394
1395 if ((vmf->flags & FAULT_FLAG_WRITE) && !vmf->cow_page)
1396 flags |= IOMAP_WRITE;
1397
1398 /*
1399 * Note that we don't bother to use iomap_apply here: DAX required
1400 * the file system block size to be equal the page size, which means
1401 * that we never have to deal with more than a single extent here.
1402 */
1403 error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap);
1404 if (error)
1405 goto unlock_entry;
1406 if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
1407 error = -EIO; /* fs corruption? */
1408 goto unlock_entry;
1409 }
1410
1411 sector = iomap.blkno + (((pos & PAGE_MASK) - iomap.offset) >> 9);
1412
1413 if (vmf->cow_page) {
1414 switch (iomap.type) {
1415 case IOMAP_HOLE:
1416 case IOMAP_UNWRITTEN:
1417 clear_user_highpage(vmf->cow_page, vaddr);
1418 break;
1419 case IOMAP_MAPPED:
1420 error = copy_user_dax(iomap.bdev, sector, PAGE_SIZE,
1421 vmf->cow_page, vaddr);
1422 break;
1423 default:
1424 WARN_ON_ONCE(1);
1425 error = -EIO;
1426 break;
1427 }
1428
1429 if (error)
1430 goto unlock_entry;
1431 if (!radix_tree_exceptional_entry(entry)) {
1432 vmf->page = entry;
1433 return VM_FAULT_LOCKED;
1434 }
1435 vmf->entry = entry;
1436 return VM_FAULT_DAX_LOCKED;
1437 }
1438
1439 switch (iomap.type) {
1440 case IOMAP_MAPPED:
1441 if (iomap.flags & IOMAP_F_NEW) {
1442 count_vm_event(PGMAJFAULT);
1443 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
1444 major = VM_FAULT_MAJOR;
1445 }
1446 error = dax_insert_mapping(mapping, iomap.bdev, sector,
1447 PAGE_SIZE, &entry, vma, vmf);
1448 break;
1449 case IOMAP_UNWRITTEN:
1450 case IOMAP_HOLE:
1451 if (!(vmf->flags & FAULT_FLAG_WRITE))
1452 return dax_load_hole(mapping, entry, vmf);
1453 /*FALLTHRU*/
1454 default:
1455 WARN_ON_ONCE(1);
1456 error = -EIO;
1457 break;
1458 }
1459
1460 unlock_entry:
1461 put_locked_mapping_entry(mapping, vmf->pgoff, entry);
1462 out:
1463 if (error == -ENOMEM)
1464 return VM_FAULT_OOM | major;
1465 /* -EBUSY is fine, somebody else faulted on the same PTE */
1466 if (error < 0 && error != -EBUSY)
1467 return VM_FAULT_SIGBUS | major;
1468 return VM_FAULT_NOPAGE | major;
1469}
1470EXPORT_SYMBOL_GPL(iomap_dax_fault);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001471#endif /* CONFIG_FS_IOMAP */