blob: 3af2da5e64ce77fa8ae4b3f294c82882d350120f [file] [log] [blame]
Matthew Wilcoxd475c632015-02-16 15:58:56 -08001/*
2 * fs/dax.c - Direct Access filesystem code
3 * Copyright (c) 2013-2014 Intel Corporation
4 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
5 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 */
16
17#include <linux/atomic.h>
18#include <linux/blkdev.h>
19#include <linux/buffer_head.h>
Ross Zwislerd77e92e2015-09-09 10:29:40 -060020#include <linux/dax.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080021#include <linux/fs.h>
22#include <linux/genhd.h>
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -080023#include <linux/highmem.h>
24#include <linux/memcontrol.h>
25#include <linux/mm.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080026#include <linux/mutex.h>
Ross Zwisler9973c982016-01-22 15:10:47 -080027#include <linux/pagevec.h>
Ross Zwisler2765cfb2015-08-18 13:55:40 -060028#include <linux/pmem.h>
Matthew Wilcox289c6ae2015-02-16 15:58:59 -080029#include <linux/sched.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080030#include <linux/uio.h>
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -080031#include <linux/vmstat.h>
Dan Williams34c0fd52016-01-15 16:56:14 -080032#include <linux/pfn_t.h>
Dan Williams0e749e52016-01-15 16:55:53 -080033#include <linux/sizes.h>
Jan Kara4b4bb462016-12-14 15:07:53 -080034#include <linux/mmu_notifier.h>
Christoph Hellwiga254e562016-09-19 11:24:49 +100035#include <linux/iomap.h>
36#include "internal.h"
Matthew Wilcoxd475c632015-02-16 15:58:56 -080037
Jan Karaac401cc2016-05-12 18:29:18 +020038/* We choose 4096 entries - same as per-zone page wait tables */
39#define DAX_WAIT_TABLE_BITS 12
40#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)
41
Ross Zwislerce95ab02016-11-08 11:31:44 +110042static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
Jan Karaac401cc2016-05-12 18:29:18 +020043
44static int __init init_dax_wait_table(void)
45{
46 int i;
47
48 for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
49 init_waitqueue_head(wait_table + i);
50 return 0;
51}
52fs_initcall(init_dax_wait_table);
53
Dan Williamsb2e0d162016-01-15 16:55:59 -080054static long dax_map_atomic(struct block_device *bdev, struct blk_dax_ctl *dax)
55{
56 struct request_queue *q = bdev->bd_queue;
57 long rc = -EIO;
58
Dan Williams7a9eb202016-06-03 18:06:47 -070059 dax->addr = ERR_PTR(-EIO);
Dan Williamsb2e0d162016-01-15 16:55:59 -080060 if (blk_queue_enter(q, true) != 0)
61 return rc;
62
63 rc = bdev_direct_access(bdev, dax);
64 if (rc < 0) {
Dan Williams7a9eb202016-06-03 18:06:47 -070065 dax->addr = ERR_PTR(rc);
Dan Williamsb2e0d162016-01-15 16:55:59 -080066 blk_queue_exit(q);
67 return rc;
68 }
69 return rc;
70}
71
72static void dax_unmap_atomic(struct block_device *bdev,
73 const struct blk_dax_ctl *dax)
74{
75 if (IS_ERR(dax->addr))
76 return;
77 blk_queue_exit(bdev->bd_queue);
78}
79
Ross Zwisler642261a2016-11-08 11:34:45 +110080static int dax_is_pmd_entry(void *entry)
81{
82 return (unsigned long)entry & RADIX_DAX_PMD;
83}
84
85static int dax_is_pte_entry(void *entry)
86{
87 return !((unsigned long)entry & RADIX_DAX_PMD);
88}
89
90static int dax_is_zero_entry(void *entry)
91{
92 return (unsigned long)entry & RADIX_DAX_HZP;
93}
94
95static int dax_is_empty_entry(void *entry)
96{
97 return (unsigned long)entry & RADIX_DAX_EMPTY;
98}
99
Dan Williamsd1a5f2b42016-01-28 20:25:31 -0800100struct page *read_dax_sector(struct block_device *bdev, sector_t n)
101{
102 struct page *page = alloc_pages(GFP_KERNEL, 0);
103 struct blk_dax_ctl dax = {
104 .size = PAGE_SIZE,
105 .sector = n & ~((((int) PAGE_SIZE) / 512) - 1),
106 };
107 long rc;
108
109 if (!page)
110 return ERR_PTR(-ENOMEM);
111
112 rc = dax_map_atomic(bdev, &dax);
113 if (rc < 0)
114 return ERR_PTR(rc);
115 memcpy_from_pmem(page_address(page), dax.addr, PAGE_SIZE);
116 dax_unmap_atomic(bdev, &dax);
117 return page;
118}
119
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800120/*
Jan Karaac401cc2016-05-12 18:29:18 +0200121 * DAX radix tree locking
122 */
123struct exceptional_entry_key {
124 struct address_space *mapping;
Ross Zwisler63e95b52016-11-08 11:32:20 +1100125 pgoff_t entry_start;
Jan Karaac401cc2016-05-12 18:29:18 +0200126};
127
128struct wait_exceptional_entry_queue {
129 wait_queue_t wait;
130 struct exceptional_entry_key key;
131};
132
Ross Zwisler63e95b52016-11-08 11:32:20 +1100133static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping,
134 pgoff_t index, void *entry, struct exceptional_entry_key *key)
135{
136 unsigned long hash;
137
138 /*
139 * If 'entry' is a PMD, align the 'index' that we use for the wait
140 * queue to the start of that PMD. This ensures that all offsets in
141 * the range covered by the PMD map to the same bit lock.
142 */
Ross Zwisler642261a2016-11-08 11:34:45 +1100143 if (dax_is_pmd_entry(entry))
Ross Zwisler63e95b52016-11-08 11:32:20 +1100144 index &= ~((1UL << (PMD_SHIFT - PAGE_SHIFT)) - 1);
145
146 key->mapping = mapping;
147 key->entry_start = index;
148
149 hash = hash_long((unsigned long)mapping ^ index, DAX_WAIT_TABLE_BITS);
150 return wait_table + hash;
151}
152
Jan Karaac401cc2016-05-12 18:29:18 +0200153static int wake_exceptional_entry_func(wait_queue_t *wait, unsigned int mode,
154 int sync, void *keyp)
155{
156 struct exceptional_entry_key *key = keyp;
157 struct wait_exceptional_entry_queue *ewait =
158 container_of(wait, struct wait_exceptional_entry_queue, wait);
159
160 if (key->mapping != ewait->key.mapping ||
Ross Zwisler63e95b52016-11-08 11:32:20 +1100161 key->entry_start != ewait->key.entry_start)
Jan Karaac401cc2016-05-12 18:29:18 +0200162 return 0;
163 return autoremove_wake_function(wait, mode, sync, NULL);
164}
165
166/*
167 * Check whether the given slot is locked. The function must be called with
168 * mapping->tree_lock held
169 */
170static inline int slot_locked(struct address_space *mapping, void **slot)
171{
172 unsigned long entry = (unsigned long)
173 radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
174 return entry & RADIX_DAX_ENTRY_LOCK;
175}
176
177/*
178 * Mark the given slot is locked. The function must be called with
179 * mapping->tree_lock held
180 */
181static inline void *lock_slot(struct address_space *mapping, void **slot)
182{
183 unsigned long entry = (unsigned long)
184 radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
185
186 entry |= RADIX_DAX_ENTRY_LOCK;
Johannes Weiner6d75f362016-12-12 16:43:43 -0800187 radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry);
Jan Karaac401cc2016-05-12 18:29:18 +0200188 return (void *)entry;
189}
190
191/*
192 * Mark the given slot is unlocked. The function must be called with
193 * mapping->tree_lock held
194 */
195static inline void *unlock_slot(struct address_space *mapping, void **slot)
196{
197 unsigned long entry = (unsigned long)
198 radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
199
200 entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK;
Johannes Weiner6d75f362016-12-12 16:43:43 -0800201 radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry);
Jan Karaac401cc2016-05-12 18:29:18 +0200202 return (void *)entry;
203}
204
205/*
206 * Lookup entry in radix tree, wait for it to become unlocked if it is
207 * exceptional entry and return it. The caller must call
208 * put_unlocked_mapping_entry() when he decided not to lock the entry or
209 * put_locked_mapping_entry() when he locked the entry and now wants to
210 * unlock it.
211 *
212 * The function must be called with mapping->tree_lock held.
213 */
214static void *get_unlocked_mapping_entry(struct address_space *mapping,
215 pgoff_t index, void ***slotp)
216{
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100217 void *entry, **slot;
Jan Karaac401cc2016-05-12 18:29:18 +0200218 struct wait_exceptional_entry_queue ewait;
Ross Zwisler63e95b52016-11-08 11:32:20 +1100219 wait_queue_head_t *wq;
Jan Karaac401cc2016-05-12 18:29:18 +0200220
221 init_wait(&ewait.wait);
222 ewait.wait.func = wake_exceptional_entry_func;
Jan Karaac401cc2016-05-12 18:29:18 +0200223
224 for (;;) {
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100225 entry = __radix_tree_lookup(&mapping->page_tree, index, NULL,
Jan Karaac401cc2016-05-12 18:29:18 +0200226 &slot);
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100227 if (!entry || !radix_tree_exceptional_entry(entry) ||
Jan Karaac401cc2016-05-12 18:29:18 +0200228 !slot_locked(mapping, slot)) {
229 if (slotp)
230 *slotp = slot;
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100231 return entry;
Jan Karaac401cc2016-05-12 18:29:18 +0200232 }
Ross Zwisler63e95b52016-11-08 11:32:20 +1100233
234 wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key);
Jan Karaac401cc2016-05-12 18:29:18 +0200235 prepare_to_wait_exclusive(wq, &ewait.wait,
236 TASK_UNINTERRUPTIBLE);
237 spin_unlock_irq(&mapping->tree_lock);
238 schedule();
239 finish_wait(wq, &ewait.wait);
240 spin_lock_irq(&mapping->tree_lock);
241 }
242}
243
Jan Karab1aa8122016-12-14 15:07:24 -0800244static void dax_unlock_mapping_entry(struct address_space *mapping,
245 pgoff_t index)
246{
247 void *entry, **slot;
248
249 spin_lock_irq(&mapping->tree_lock);
250 entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot);
251 if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) ||
252 !slot_locked(mapping, slot))) {
253 spin_unlock_irq(&mapping->tree_lock);
254 return;
255 }
256 unlock_slot(mapping, slot);
257 spin_unlock_irq(&mapping->tree_lock);
258 dax_wake_mapping_entry_waiter(mapping, index, entry, false);
259}
260
Jan Karaac401cc2016-05-12 18:29:18 +0200261static void put_locked_mapping_entry(struct address_space *mapping,
262 pgoff_t index, void *entry)
263{
264 if (!radix_tree_exceptional_entry(entry)) {
265 unlock_page(entry);
266 put_page(entry);
267 } else {
Jan Karabc2466e2016-05-12 18:29:19 +0200268 dax_unlock_mapping_entry(mapping, index);
Jan Karaac401cc2016-05-12 18:29:18 +0200269 }
270}
271
272/*
273 * Called when we are done with radix tree entry we looked up via
274 * get_unlocked_mapping_entry() and which we didn't lock in the end.
275 */
276static void put_unlocked_mapping_entry(struct address_space *mapping,
277 pgoff_t index, void *entry)
278{
279 if (!radix_tree_exceptional_entry(entry))
280 return;
281
282 /* We have to wake up next waiter for the radix tree entry lock */
Ross Zwisler422476c2016-11-08 11:33:44 +1100283 dax_wake_mapping_entry_waiter(mapping, index, entry, false);
284}
285
Jan Karaac401cc2016-05-12 18:29:18 +0200286/*
287 * Find radix tree entry at given index. If it points to a page, return with
288 * the page locked. If it points to the exceptional entry, return with the
289 * radix tree entry locked. If the radix tree doesn't contain given index,
290 * create empty exceptional entry for the index and return with it locked.
291 *
Ross Zwisler642261a2016-11-08 11:34:45 +1100292 * When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will
293 * either return that locked entry or will return an error. This error will
294 * happen if there are any 4k entries (either zero pages or DAX entries)
295 * within the 2MiB range that we are requesting.
296 *
297 * We always favor 4k entries over 2MiB entries. There isn't a flow where we
298 * evict 4k entries in order to 'upgrade' them to a 2MiB entry. A 2MiB
299 * insertion will fail if it finds any 4k entries already in the tree, and a
300 * 4k insertion will cause an existing 2MiB entry to be unmapped and
301 * downgraded to 4k entries. This happens for both 2MiB huge zero pages as
302 * well as 2MiB empty entries.
303 *
304 * The exception to this downgrade path is for 2MiB DAX PMD entries that have
305 * real storage backing them. We will leave these real 2MiB DAX entries in
306 * the tree, and PTE writes will simply dirty the entire 2MiB DAX entry.
307 *
Jan Karaac401cc2016-05-12 18:29:18 +0200308 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
309 * persistent memory the benefit is doubtful. We can add that later if we can
310 * show it helps.
311 */
Ross Zwisler642261a2016-11-08 11:34:45 +1100312static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index,
313 unsigned long size_flag)
Jan Karaac401cc2016-05-12 18:29:18 +0200314{
Ross Zwisler642261a2016-11-08 11:34:45 +1100315 bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100316 void *entry, **slot;
Jan Karaac401cc2016-05-12 18:29:18 +0200317
318restart:
319 spin_lock_irq(&mapping->tree_lock);
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100320 entry = get_unlocked_mapping_entry(mapping, index, &slot);
Ross Zwisler642261a2016-11-08 11:34:45 +1100321
322 if (entry) {
323 if (size_flag & RADIX_DAX_PMD) {
324 if (!radix_tree_exceptional_entry(entry) ||
325 dax_is_pte_entry(entry)) {
326 put_unlocked_mapping_entry(mapping, index,
327 entry);
328 entry = ERR_PTR(-EEXIST);
329 goto out_unlock;
330 }
331 } else { /* trying to grab a PTE entry */
332 if (radix_tree_exceptional_entry(entry) &&
333 dax_is_pmd_entry(entry) &&
334 (dax_is_zero_entry(entry) ||
335 dax_is_empty_entry(entry))) {
336 pmd_downgrade = true;
337 }
338 }
339 }
340
Jan Karaac401cc2016-05-12 18:29:18 +0200341 /* No entry for given index? Make sure radix tree is big enough. */
Ross Zwisler642261a2016-11-08 11:34:45 +1100342 if (!entry || pmd_downgrade) {
Jan Karaac401cc2016-05-12 18:29:18 +0200343 int err;
344
Ross Zwisler642261a2016-11-08 11:34:45 +1100345 if (pmd_downgrade) {
346 /*
347 * Make sure 'entry' remains valid while we drop
348 * mapping->tree_lock.
349 */
350 entry = lock_slot(mapping, slot);
351 }
352
Jan Karaac401cc2016-05-12 18:29:18 +0200353 spin_unlock_irq(&mapping->tree_lock);
Ross Zwisler642261a2016-11-08 11:34:45 +1100354 /*
355 * Besides huge zero pages the only other thing that gets
356 * downgraded are empty entries which don't need to be
357 * unmapped.
358 */
359 if (pmd_downgrade && dax_is_zero_entry(entry))
360 unmap_mapping_range(mapping,
361 (index << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0);
362
Jan Kara0cb80b42016-12-12 21:34:12 -0500363 err = radix_tree_preload(
364 mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM);
365 if (err) {
366 if (pmd_downgrade)
367 put_locked_mapping_entry(mapping, index, entry);
368 return ERR_PTR(err);
369 }
Jan Karaac401cc2016-05-12 18:29:18 +0200370 spin_lock_irq(&mapping->tree_lock);
Ross Zwisler642261a2016-11-08 11:34:45 +1100371
372 if (pmd_downgrade) {
373 radix_tree_delete(&mapping->page_tree, index);
374 mapping->nrexceptional--;
375 dax_wake_mapping_entry_waiter(mapping, index, entry,
376 true);
377 }
378
379 entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY);
380
381 err = __radix_tree_insert(&mapping->page_tree, index,
382 dax_radix_order(entry), entry);
Jan Karaac401cc2016-05-12 18:29:18 +0200383 radix_tree_preload_end();
384 if (err) {
385 spin_unlock_irq(&mapping->tree_lock);
Ross Zwisler642261a2016-11-08 11:34:45 +1100386 /*
387 * Someone already created the entry? This is a
388 * normal failure when inserting PMDs in a range
389 * that already contains PTEs. In that case we want
390 * to return -EEXIST immediately.
391 */
392 if (err == -EEXIST && !(size_flag & RADIX_DAX_PMD))
Jan Karaac401cc2016-05-12 18:29:18 +0200393 goto restart;
Ross Zwisler642261a2016-11-08 11:34:45 +1100394 /*
395 * Our insertion of a DAX PMD entry failed, most
396 * likely because it collided with a PTE sized entry
397 * at a different index in the PMD range. We haven't
398 * inserted anything into the radix tree and have no
399 * waiters to wake.
400 */
Jan Karaac401cc2016-05-12 18:29:18 +0200401 return ERR_PTR(err);
402 }
403 /* Good, we have inserted empty locked entry into the tree. */
404 mapping->nrexceptional++;
405 spin_unlock_irq(&mapping->tree_lock);
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100406 return entry;
Jan Karaac401cc2016-05-12 18:29:18 +0200407 }
408 /* Normal page in radix tree? */
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100409 if (!radix_tree_exceptional_entry(entry)) {
410 struct page *page = entry;
Jan Karaac401cc2016-05-12 18:29:18 +0200411
412 get_page(page);
413 spin_unlock_irq(&mapping->tree_lock);
414 lock_page(page);
415 /* Page got truncated? Retry... */
416 if (unlikely(page->mapping != mapping)) {
417 unlock_page(page);
418 put_page(page);
419 goto restart;
420 }
421 return page;
422 }
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100423 entry = lock_slot(mapping, slot);
Ross Zwisler642261a2016-11-08 11:34:45 +1100424 out_unlock:
Jan Karaac401cc2016-05-12 18:29:18 +0200425 spin_unlock_irq(&mapping->tree_lock);
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100426 return entry;
Jan Karaac401cc2016-05-12 18:29:18 +0200427}
428
Ross Zwisler63e95b52016-11-08 11:32:20 +1100429/*
430 * We do not necessarily hold the mapping->tree_lock when we call this
431 * function so it is possible that 'entry' is no longer a valid item in the
Ross Zwisler642261a2016-11-08 11:34:45 +1100432 * radix tree. This is okay because all we really need to do is to find the
433 * correct waitqueue where tasks might be waiting for that old 'entry' and
434 * wake them.
Ross Zwisler63e95b52016-11-08 11:32:20 +1100435 */
Jan Karaac401cc2016-05-12 18:29:18 +0200436void dax_wake_mapping_entry_waiter(struct address_space *mapping,
Ross Zwisler63e95b52016-11-08 11:32:20 +1100437 pgoff_t index, void *entry, bool wake_all)
Jan Karaac401cc2016-05-12 18:29:18 +0200438{
Ross Zwisler63e95b52016-11-08 11:32:20 +1100439 struct exceptional_entry_key key;
440 wait_queue_head_t *wq;
441
442 wq = dax_entry_waitqueue(mapping, index, entry, &key);
Jan Karaac401cc2016-05-12 18:29:18 +0200443
444 /*
445 * Checking for locked entry and prepare_to_wait_exclusive() happens
446 * under mapping->tree_lock, ditto for entry handling in our callers.
447 * So at this point all tasks that could have seen our entry locked
448 * must be in the waitqueue and the following check will see them.
449 */
Ross Zwisler63e95b52016-11-08 11:32:20 +1100450 if (waitqueue_active(wq))
Jan Karaac401cc2016-05-12 18:29:18 +0200451 __wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
Jan Karaac401cc2016-05-12 18:29:18 +0200452}
453
Jan Karac6dcf522016-08-10 17:22:44 +0200454static int __dax_invalidate_mapping_entry(struct address_space *mapping,
455 pgoff_t index, bool trunc)
456{
457 int ret = 0;
458 void *entry;
459 struct radix_tree_root *page_tree = &mapping->page_tree;
460
461 spin_lock_irq(&mapping->tree_lock);
462 entry = get_unlocked_mapping_entry(mapping, index, NULL);
463 if (!entry || !radix_tree_exceptional_entry(entry))
464 goto out;
465 if (!trunc &&
466 (radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) ||
467 radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)))
468 goto out;
469 radix_tree_delete(page_tree, index);
470 mapping->nrexceptional--;
471 ret = 1;
472out:
473 put_unlocked_mapping_entry(mapping, index, entry);
474 spin_unlock_irq(&mapping->tree_lock);
475 return ret;
476}
Jan Karaac401cc2016-05-12 18:29:18 +0200477/*
478 * Delete exceptional DAX entry at @index from @mapping. Wait for radix tree
479 * entry to get unlocked before deleting it.
480 */
481int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
482{
Jan Karac6dcf522016-08-10 17:22:44 +0200483 int ret = __dax_invalidate_mapping_entry(mapping, index, true);
Jan Karaac401cc2016-05-12 18:29:18 +0200484
Jan Karaac401cc2016-05-12 18:29:18 +0200485 /*
486 * This gets called from truncate / punch_hole path. As such, the caller
487 * must hold locks protecting against concurrent modifications of the
488 * radix tree (usually fs-private i_mmap_sem for writing). Since the
489 * caller has seen exceptional entry for this index, we better find it
490 * at that index as well...
491 */
Jan Karac6dcf522016-08-10 17:22:44 +0200492 WARN_ON_ONCE(!ret);
493 return ret;
494}
Jan Karaac401cc2016-05-12 18:29:18 +0200495
Jan Karac6dcf522016-08-10 17:22:44 +0200496/*
497 * Invalidate exceptional DAX entry if easily possible. This handles DAX
498 * entries for invalidate_inode_pages() so we evict the entry only if we can
499 * do so without blocking.
500 */
501int dax_invalidate_mapping_entry(struct address_space *mapping, pgoff_t index)
502{
503 int ret = 0;
504 void *entry, **slot;
505 struct radix_tree_root *page_tree = &mapping->page_tree;
506
507 spin_lock_irq(&mapping->tree_lock);
508 entry = __radix_tree_lookup(page_tree, index, NULL, &slot);
509 if (!entry || !radix_tree_exceptional_entry(entry) ||
510 slot_locked(mapping, slot))
511 goto out;
512 if (radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) ||
513 radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))
514 goto out;
515 radix_tree_delete(page_tree, index);
516 mapping->nrexceptional--;
517 ret = 1;
518out:
519 spin_unlock_irq(&mapping->tree_lock);
520 if (ret)
521 dax_wake_mapping_entry_waiter(mapping, index, entry, true);
522 return ret;
523}
524
525/*
526 * Invalidate exceptional DAX entry if it is clean.
527 */
528int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
529 pgoff_t index)
530{
531 return __dax_invalidate_mapping_entry(mapping, index, false);
Jan Karaac401cc2016-05-12 18:29:18 +0200532}
533
534/*
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800535 * The user has performed a load from a hole in the file. Allocating
536 * a new page in the file would cause excessive storage usage for
537 * workloads with sparse files. We allocate a page cache page instead.
538 * We'll kick it out of the page cache if it's ever written to,
539 * otherwise it will simply fall out of the page cache under memory
540 * pressure without ever having been dirtied.
541 */
Jan Karaf449b932016-10-19 14:48:38 +0200542static int dax_load_hole(struct address_space *mapping, void **entry,
Jan Karaac401cc2016-05-12 18:29:18 +0200543 struct vm_fault *vmf)
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800544{
Jan Karaac401cc2016-05-12 18:29:18 +0200545 struct page *page;
Jan Karaf449b932016-10-19 14:48:38 +0200546 int ret;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800547
Jan Karaac401cc2016-05-12 18:29:18 +0200548 /* Hole page already exists? Return it... */
Jan Karaf449b932016-10-19 14:48:38 +0200549 if (!radix_tree_exceptional_entry(*entry)) {
550 page = *entry;
551 goto out;
Jan Karaac401cc2016-05-12 18:29:18 +0200552 }
553
554 /* This will replace locked radix tree entry with a hole page */
555 page = find_or_create_page(mapping, vmf->pgoff,
556 vmf->gfp_mask | __GFP_ZERO);
Jan Karab1aa8122016-12-14 15:07:24 -0800557 if (!page)
Jan Karaac401cc2016-05-12 18:29:18 +0200558 return VM_FAULT_OOM;
Jan Karaf449b932016-10-19 14:48:38 +0200559 out:
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800560 vmf->page = page;
Jan Karaf449b932016-10-19 14:48:38 +0200561 ret = finish_fault(vmf);
562 vmf->page = NULL;
563 *entry = page;
564 if (!ret) {
565 /* Grab reference for PTE that is now referencing the page */
566 get_page(page);
567 return VM_FAULT_NOPAGE;
568 }
569 return ret;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800570}
571
Christoph Hellwigb0d5e822016-09-19 11:24:49 +1000572static int copy_user_dax(struct block_device *bdev, sector_t sector, size_t size,
573 struct page *to, unsigned long vaddr)
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800574{
Dan Williamsb2e0d162016-01-15 16:55:59 -0800575 struct blk_dax_ctl dax = {
Christoph Hellwigb0d5e822016-09-19 11:24:49 +1000576 .sector = sector,
577 .size = size,
Dan Williamsb2e0d162016-01-15 16:55:59 -0800578 };
Ross Zwislere2e05392015-08-18 13:55:41 -0600579 void *vto;
580
Dan Williamsb2e0d162016-01-15 16:55:59 -0800581 if (dax_map_atomic(bdev, &dax) < 0)
582 return PTR_ERR(dax.addr);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800583 vto = kmap_atomic(to);
Dan Williamsb2e0d162016-01-15 16:55:59 -0800584 copy_user_page(vto, (void __force *)dax.addr, vaddr, to);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800585 kunmap_atomic(vto);
Dan Williamsb2e0d162016-01-15 16:55:59 -0800586 dax_unmap_atomic(bdev, &dax);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800587 return 0;
588}
589
Ross Zwisler642261a2016-11-08 11:34:45 +1100590/*
591 * By this point grab_mapping_entry() has ensured that we have a locked entry
592 * of the appropriate size so we don't have to worry about downgrading PMDs to
593 * PTEs. If we happen to be trying to insert a PTE and there is a PMD
594 * already in the tree, we will skip the insertion and just dirty the PMD as
595 * appropriate.
596 */
Jan Karaac401cc2016-05-12 18:29:18 +0200597static void *dax_insert_mapping_entry(struct address_space *mapping,
598 struct vm_fault *vmf,
Ross Zwisler642261a2016-11-08 11:34:45 +1100599 void *entry, sector_t sector,
600 unsigned long flags)
Ross Zwisler9973c982016-01-22 15:10:47 -0800601{
602 struct radix_tree_root *page_tree = &mapping->page_tree;
Jan Karaac401cc2016-05-12 18:29:18 +0200603 int error = 0;
604 bool hole_fill = false;
605 void *new_entry;
606 pgoff_t index = vmf->pgoff;
Ross Zwisler9973c982016-01-22 15:10:47 -0800607
Jan Karaac401cc2016-05-12 18:29:18 +0200608 if (vmf->flags & FAULT_FLAG_WRITE)
Dmitry Monakhovd2b2a282016-02-05 15:36:55 -0800609 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
Ross Zwisler9973c982016-01-22 15:10:47 -0800610
Jan Karaac401cc2016-05-12 18:29:18 +0200611 /* Replacing hole page with block mapping? */
612 if (!radix_tree_exceptional_entry(entry)) {
613 hole_fill = true;
614 /*
615 * Unmap the page now before we remove it from page cache below.
616 * The page is locked so it cannot be faulted in again.
617 */
618 unmap_mapping_range(mapping, vmf->pgoff << PAGE_SHIFT,
619 PAGE_SIZE, 0);
620 error = radix_tree_preload(vmf->gfp_mask & ~__GFP_HIGHMEM);
621 if (error)
622 return ERR_PTR(error);
Ross Zwisler642261a2016-11-08 11:34:45 +1100623 } else if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_HZP)) {
624 /* replacing huge zero page with PMD block mapping */
625 unmap_mapping_range(mapping,
626 (vmf->pgoff << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0);
Ross Zwisler9973c982016-01-22 15:10:47 -0800627 }
628
Jan Karaac401cc2016-05-12 18:29:18 +0200629 spin_lock_irq(&mapping->tree_lock);
Ross Zwisler642261a2016-11-08 11:34:45 +1100630 new_entry = dax_radix_locked_entry(sector, flags);
631
Jan Karaac401cc2016-05-12 18:29:18 +0200632 if (hole_fill) {
633 __delete_from_page_cache(entry, NULL);
634 /* Drop pagecache reference */
635 put_page(entry);
Ross Zwisler642261a2016-11-08 11:34:45 +1100636 error = __radix_tree_insert(page_tree, index,
637 dax_radix_order(new_entry), new_entry);
Jan Karaac401cc2016-05-12 18:29:18 +0200638 if (error) {
639 new_entry = ERR_PTR(error);
Ross Zwisler9973c982016-01-22 15:10:47 -0800640 goto unlock;
641 }
Jan Karaac401cc2016-05-12 18:29:18 +0200642 mapping->nrexceptional++;
Ross Zwisler642261a2016-11-08 11:34:45 +1100643 } else if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
644 /*
645 * Only swap our new entry into the radix tree if the current
646 * entry is a zero page or an empty entry. If a normal PTE or
647 * PMD entry is already in the tree, we leave it alone. This
648 * means that if we are trying to insert a PTE and the
649 * existing entry is a PMD, we will just leave the PMD in the
650 * tree and dirty it if necessary.
651 */
Johannes Weinerf7942432016-12-12 16:43:41 -0800652 struct radix_tree_node *node;
Jan Karaac401cc2016-05-12 18:29:18 +0200653 void **slot;
654 void *ret;
Ross Zwisler9973c982016-01-22 15:10:47 -0800655
Johannes Weinerf7942432016-12-12 16:43:41 -0800656 ret = __radix_tree_lookup(page_tree, index, &node, &slot);
Jan Karaac401cc2016-05-12 18:29:18 +0200657 WARN_ON_ONCE(ret != entry);
Johannes Weiner4d693d02016-12-12 16:43:49 -0800658 __radix_tree_replace(page_tree, node, slot,
659 new_entry, NULL, NULL);
Ross Zwisler9973c982016-01-22 15:10:47 -0800660 }
Jan Karaac401cc2016-05-12 18:29:18 +0200661 if (vmf->flags & FAULT_FLAG_WRITE)
Ross Zwisler9973c982016-01-22 15:10:47 -0800662 radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY);
663 unlock:
664 spin_unlock_irq(&mapping->tree_lock);
Jan Karaac401cc2016-05-12 18:29:18 +0200665 if (hole_fill) {
666 radix_tree_preload_end();
667 /*
668 * We don't need hole page anymore, it has been replaced with
669 * locked radix tree entry now.
670 */
671 if (mapping->a_ops->freepage)
672 mapping->a_ops->freepage(entry);
673 unlock_page(entry);
674 put_page(entry);
675 }
676 return new_entry;
Ross Zwisler9973c982016-01-22 15:10:47 -0800677}
678
Jan Kara4b4bb462016-12-14 15:07:53 -0800679static inline unsigned long
680pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
681{
682 unsigned long address;
683
684 address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
685 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
686 return address;
687}
688
689/* Walk all mappings of a given index of a file and writeprotect them */
690static void dax_mapping_entry_mkclean(struct address_space *mapping,
691 pgoff_t index, unsigned long pfn)
692{
693 struct vm_area_struct *vma;
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800694 pte_t pte, *ptep = NULL;
695 pmd_t *pmdp = NULL;
Jan Kara4b4bb462016-12-14 15:07:53 -0800696 spinlock_t *ptl;
697 bool changed;
698
699 i_mmap_lock_read(mapping);
700 vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
701 unsigned long address;
702
703 cond_resched();
704
705 if (!(vma->vm_flags & VM_SHARED))
706 continue;
707
708 address = pgoff_address(index, vma);
709 changed = false;
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800710 if (follow_pte_pmd(vma->vm_mm, address, &ptep, &pmdp, &ptl))
Jan Kara4b4bb462016-12-14 15:07:53 -0800711 continue;
Jan Kara4b4bb462016-12-14 15:07:53 -0800712
Ross Zwislerf729c8c2017-01-10 16:57:24 -0800713 if (pmdp) {
714#ifdef CONFIG_FS_DAX_PMD
715 pmd_t pmd;
716
717 if (pfn != pmd_pfn(*pmdp))
718 goto unlock_pmd;
719 if (!pmd_dirty(*pmdp) && !pmd_write(*pmdp))
720 goto unlock_pmd;
721
722 flush_cache_page(vma, address, pfn);
723 pmd = pmdp_huge_clear_flush(vma, address, pmdp);
724 pmd = pmd_wrprotect(pmd);
725 pmd = pmd_mkclean(pmd);
726 set_pmd_at(vma->vm_mm, address, pmdp, pmd);
727 changed = true;
728unlock_pmd:
729 spin_unlock(ptl);
730#endif
731 } else {
732 if (pfn != pte_pfn(*ptep))
733 goto unlock_pte;
734 if (!pte_dirty(*ptep) && !pte_write(*ptep))
735 goto unlock_pte;
736
737 flush_cache_page(vma, address, pfn);
738 pte = ptep_clear_flush(vma, address, ptep);
739 pte = pte_wrprotect(pte);
740 pte = pte_mkclean(pte);
741 set_pte_at(vma->vm_mm, address, ptep, pte);
742 changed = true;
743unlock_pte:
744 pte_unmap_unlock(ptep, ptl);
745 }
Jan Kara4b4bb462016-12-14 15:07:53 -0800746
747 if (changed)
748 mmu_notifier_invalidate_page(vma->vm_mm, address);
749 }
750 i_mmap_unlock_read(mapping);
751}
752
Ross Zwisler9973c982016-01-22 15:10:47 -0800753static int dax_writeback_one(struct block_device *bdev,
754 struct address_space *mapping, pgoff_t index, void *entry)
755{
756 struct radix_tree_root *page_tree = &mapping->page_tree;
Ross Zwisler9973c982016-01-22 15:10:47 -0800757 struct blk_dax_ctl dax;
Jan Karaa6abc2c2016-12-14 15:07:47 -0800758 void *entry2, **slot;
Ross Zwisler9973c982016-01-22 15:10:47 -0800759 int ret = 0;
760
Ross Zwisler9973c982016-01-22 15:10:47 -0800761 /*
Jan Karaa6abc2c2016-12-14 15:07:47 -0800762 * A page got tagged dirty in DAX mapping? Something is seriously
763 * wrong.
Ross Zwisler9973c982016-01-22 15:10:47 -0800764 */
Jan Karaa6abc2c2016-12-14 15:07:47 -0800765 if (WARN_ON(!radix_tree_exceptional_entry(entry)))
766 return -EIO;
Ross Zwisler9973c982016-01-22 15:10:47 -0800767
Jan Karaa6abc2c2016-12-14 15:07:47 -0800768 spin_lock_irq(&mapping->tree_lock);
769 entry2 = get_unlocked_mapping_entry(mapping, index, &slot);
770 /* Entry got punched out / reallocated? */
771 if (!entry2 || !radix_tree_exceptional_entry(entry2))
772 goto put_unlocked;
773 /*
774 * Entry got reallocated elsewhere? No need to writeback. We have to
775 * compare sectors as we must not bail out due to difference in lockbit
776 * or entry type.
777 */
778 if (dax_radix_sector(entry2) != dax_radix_sector(entry))
779 goto put_unlocked;
Ross Zwisler642261a2016-11-08 11:34:45 +1100780 if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
781 dax_is_zero_entry(entry))) {
Ross Zwisler9973c982016-01-22 15:10:47 -0800782 ret = -EIO;
Jan Karaa6abc2c2016-12-14 15:07:47 -0800783 goto put_unlocked;
Ross Zwisler9973c982016-01-22 15:10:47 -0800784 }
785
Jan Karaa6abc2c2016-12-14 15:07:47 -0800786 /* Another fsync thread may have already written back this entry */
787 if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))
788 goto put_unlocked;
789 /* Lock the entry to serialize with page faults */
790 entry = lock_slot(mapping, slot);
791 /*
792 * We can clear the tag now but we have to be careful so that concurrent
793 * dax_writeback_one() calls for the same index cannot finish before we
794 * actually flush the caches. This is achieved as the calls will look
795 * at the entry only under tree_lock and once they do that they will
796 * see the entry locked and wait for it to unlock.
797 */
798 radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE);
799 spin_unlock_irq(&mapping->tree_lock);
800
Ross Zwisler642261a2016-11-08 11:34:45 +1100801 /*
802 * Even if dax_writeback_mapping_range() was given a wbc->range_start
803 * in the middle of a PMD, the 'index' we are given will be aligned to
804 * the start index of the PMD, as will the sector we pull from
805 * 'entry'. This allows us to flush for PMD_SIZE and not have to
806 * worry about partial PMD writebacks.
807 */
808 dax.sector = dax_radix_sector(entry);
809 dax.size = PAGE_SIZE << dax_radix_order(entry);
Ross Zwisler9973c982016-01-22 15:10:47 -0800810
811 /*
812 * We cannot hold tree_lock while calling dax_map_atomic() because it
813 * eventually calls cond_resched().
814 */
815 ret = dax_map_atomic(bdev, &dax);
Jan Karaa6abc2c2016-12-14 15:07:47 -0800816 if (ret < 0) {
817 put_locked_mapping_entry(mapping, index, entry);
Ross Zwisler9973c982016-01-22 15:10:47 -0800818 return ret;
Jan Karaa6abc2c2016-12-14 15:07:47 -0800819 }
Ross Zwisler9973c982016-01-22 15:10:47 -0800820
821 if (WARN_ON_ONCE(ret < dax.size)) {
822 ret = -EIO;
823 goto unmap;
824 }
825
Jan Kara4b4bb462016-12-14 15:07:53 -0800826 dax_mapping_entry_mkclean(mapping, index, pfn_t_to_pfn(dax.pfn));
Ross Zwisler9973c982016-01-22 15:10:47 -0800827 wb_cache_pmem(dax.addr, dax.size);
Jan Kara4b4bb462016-12-14 15:07:53 -0800828 /*
829 * After we have flushed the cache, we can clear the dirty tag. There
830 * cannot be new dirty data in the pfn after the flush has completed as
831 * the pfn mappings are writeprotected and fault waits for mapping
832 * entry lock.
833 */
834 spin_lock_irq(&mapping->tree_lock);
835 radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_DIRTY);
836 spin_unlock_irq(&mapping->tree_lock);
Ross Zwisler9973c982016-01-22 15:10:47 -0800837 unmap:
838 dax_unmap_atomic(bdev, &dax);
Jan Karaa6abc2c2016-12-14 15:07:47 -0800839 put_locked_mapping_entry(mapping, index, entry);
Ross Zwisler9973c982016-01-22 15:10:47 -0800840 return ret;
841
Jan Karaa6abc2c2016-12-14 15:07:47 -0800842 put_unlocked:
843 put_unlocked_mapping_entry(mapping, index, entry2);
Ross Zwisler9973c982016-01-22 15:10:47 -0800844 spin_unlock_irq(&mapping->tree_lock);
845 return ret;
846}
847
848/*
849 * Flush the mapping to the persistent domain within the byte range of [start,
850 * end]. This is required by data integrity operations to ensure file data is
851 * on persistent storage prior to completion of the operation.
852 */
Ross Zwisler7f6d5b52016-02-26 15:19:55 -0800853int dax_writeback_mapping_range(struct address_space *mapping,
854 struct block_device *bdev, struct writeback_control *wbc)
Ross Zwisler9973c982016-01-22 15:10:47 -0800855{
856 struct inode *inode = mapping->host;
Ross Zwisler642261a2016-11-08 11:34:45 +1100857 pgoff_t start_index, end_index;
Ross Zwisler9973c982016-01-22 15:10:47 -0800858 pgoff_t indices[PAGEVEC_SIZE];
859 struct pagevec pvec;
860 bool done = false;
861 int i, ret = 0;
Ross Zwisler9973c982016-01-22 15:10:47 -0800862
863 if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
864 return -EIO;
865
Ross Zwisler7f6d5b52016-02-26 15:19:55 -0800866 if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
867 return 0;
868
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300869 start_index = wbc->range_start >> PAGE_SHIFT;
870 end_index = wbc->range_end >> PAGE_SHIFT;
Ross Zwisler9973c982016-01-22 15:10:47 -0800871
872 tag_pages_for_writeback(mapping, start_index, end_index);
873
874 pagevec_init(&pvec, 0);
875 while (!done) {
876 pvec.nr = find_get_entries_tag(mapping, start_index,
877 PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE,
878 pvec.pages, indices);
879
880 if (pvec.nr == 0)
881 break;
882
883 for (i = 0; i < pvec.nr; i++) {
884 if (indices[i] > end_index) {
885 done = true;
886 break;
887 }
888
889 ret = dax_writeback_one(bdev, mapping, indices[i],
890 pvec.pages[i]);
891 if (ret < 0)
892 return ret;
893 }
894 }
Ross Zwisler9973c982016-01-22 15:10:47 -0800895 return 0;
896}
897EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);
898
Jan Karaac401cc2016-05-12 18:29:18 +0200899static int dax_insert_mapping(struct address_space *mapping,
Christoph Hellwig1aaba092016-09-19 11:24:49 +1000900 struct block_device *bdev, sector_t sector, size_t size,
901 void **entryp, struct vm_area_struct *vma, struct vm_fault *vmf)
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800902{
Jan Kara1a29d852016-12-14 15:07:01 -0800903 unsigned long vaddr = vmf->address;
Dan Williamsb2e0d162016-01-15 16:55:59 -0800904 struct blk_dax_ctl dax = {
Christoph Hellwig1aaba092016-09-19 11:24:49 +1000905 .sector = sector,
906 .size = size,
Dan Williamsb2e0d162016-01-15 16:55:59 -0800907 };
Jan Karaac401cc2016-05-12 18:29:18 +0200908 void *ret;
909 void *entry = *entryp;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800910
Jan Kara4d9a2c82016-05-12 18:29:20 +0200911 if (dax_map_atomic(bdev, &dax) < 0)
912 return PTR_ERR(dax.addr);
Dan Williamsb2e0d162016-01-15 16:55:59 -0800913 dax_unmap_atomic(bdev, &dax);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800914
Ross Zwisler642261a2016-11-08 11:34:45 +1100915 ret = dax_insert_mapping_entry(mapping, vmf, entry, dax.sector, 0);
Jan Kara4d9a2c82016-05-12 18:29:20 +0200916 if (IS_ERR(ret))
917 return PTR_ERR(ret);
Jan Karaac401cc2016-05-12 18:29:18 +0200918 *entryp = ret;
Ross Zwisler9973c982016-01-22 15:10:47 -0800919
Jan Kara4d9a2c82016-05-12 18:29:20 +0200920 return vm_insert_mixed(vma, vaddr, dax.pfn);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800921}
922
Dave Chinnerce5c5d52015-06-04 09:18:18 +1000923/**
Boaz Harrosh0e3b2102015-04-15 16:15:14 -0700924 * dax_pfn_mkwrite - handle first write to DAX page
925 * @vma: The virtual memory area where the fault occurred
926 * @vmf: The description of the fault
Boaz Harrosh0e3b2102015-04-15 16:15:14 -0700927 */
928int dax_pfn_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
929{
Ross Zwisler9973c982016-01-22 15:10:47 -0800930 struct file *file = vma->vm_file;
Jan Karaac401cc2016-05-12 18:29:18 +0200931 struct address_space *mapping = file->f_mapping;
Jan Kara2f89dc12016-12-14 15:07:50 -0800932 void *entry, **slot;
Jan Karaac401cc2016-05-12 18:29:18 +0200933 pgoff_t index = vmf->pgoff;
Boaz Harrosh0e3b2102015-04-15 16:15:14 -0700934
Jan Karaac401cc2016-05-12 18:29:18 +0200935 spin_lock_irq(&mapping->tree_lock);
Jan Kara2f89dc12016-12-14 15:07:50 -0800936 entry = get_unlocked_mapping_entry(mapping, index, &slot);
937 if (!entry || !radix_tree_exceptional_entry(entry)) {
938 if (entry)
939 put_unlocked_mapping_entry(mapping, index, entry);
940 spin_unlock_irq(&mapping->tree_lock);
941 return VM_FAULT_NOPAGE;
942 }
Jan Karaac401cc2016-05-12 18:29:18 +0200943 radix_tree_tag_set(&mapping->page_tree, index, PAGECACHE_TAG_DIRTY);
Jan Kara2f89dc12016-12-14 15:07:50 -0800944 entry = lock_slot(mapping, slot);
Jan Karaac401cc2016-05-12 18:29:18 +0200945 spin_unlock_irq(&mapping->tree_lock);
Jan Kara2f89dc12016-12-14 15:07:50 -0800946 /*
947 * If we race with somebody updating the PTE and finish_mkwrite_fault()
948 * fails, we don't care. We need to return VM_FAULT_NOPAGE and retry
949 * the fault in either case.
950 */
951 finish_mkwrite_fault(vmf);
952 put_locked_mapping_entry(mapping, index, entry);
Boaz Harrosh0e3b2102015-04-15 16:15:14 -0700953 return VM_FAULT_NOPAGE;
954}
955EXPORT_SYMBOL_GPL(dax_pfn_mkwrite);
956
Vishal Verma4b0228f2016-04-21 15:13:46 -0400957static bool dax_range_is_aligned(struct block_device *bdev,
958 unsigned int offset, unsigned int length)
959{
960 unsigned short sector_size = bdev_logical_block_size(bdev);
961
962 if (!IS_ALIGNED(offset, sector_size))
963 return false;
964 if (!IS_ALIGNED(length, sector_size))
965 return false;
966
967 return true;
968}
969
Christoph Hellwig679c8bd2016-05-09 10:47:04 +0200970int __dax_zero_page_range(struct block_device *bdev, sector_t sector,
971 unsigned int offset, unsigned int length)
972{
973 struct blk_dax_ctl dax = {
974 .sector = sector,
975 .size = PAGE_SIZE,
976 };
977
Vishal Verma4b0228f2016-04-21 15:13:46 -0400978 if (dax_range_is_aligned(bdev, offset, length)) {
979 sector_t start_sector = dax.sector + (offset >> 9);
980
981 return blkdev_issue_zeroout(bdev, start_sector,
982 length >> 9, GFP_NOFS, true);
983 } else {
984 if (dax_map_atomic(bdev, &dax) < 0)
985 return PTR_ERR(dax.addr);
986 clear_pmem(dax.addr + offset, length);
Vishal Verma4b0228f2016-04-21 15:13:46 -0400987 dax_unmap_atomic(bdev, &dax);
988 }
Christoph Hellwig679c8bd2016-05-09 10:47:04 +0200989 return 0;
990}
991EXPORT_SYMBOL_GPL(__dax_zero_page_range);
992
Ross Zwisler333ccc92016-11-08 11:33:09 +1100993static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)
994{
995 return iomap->blkno + (((pos & PAGE_MASK) - iomap->offset) >> 9);
996}
997
Christoph Hellwiga254e562016-09-19 11:24:49 +1000998static loff_t
Ross Zwisler11c59c92016-11-08 11:32:46 +1100999dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
Christoph Hellwiga254e562016-09-19 11:24:49 +10001000 struct iomap *iomap)
1001{
1002 struct iov_iter *iter = data;
1003 loff_t end = pos + length, done = 0;
1004 ssize_t ret = 0;
1005
1006 if (iov_iter_rw(iter) == READ) {
1007 end = min(end, i_size_read(inode));
1008 if (pos >= end)
1009 return 0;
1010
1011 if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
1012 return iov_iter_zero(min(length, end - pos), iter);
1013 }
1014
1015 if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))
1016 return -EIO;
1017
Jan Karae3fce682016-08-10 17:10:28 +02001018 /*
1019 * Write can allocate block for an area which has a hole page mapped
1020 * into page tables. We have to tear down these mappings so that data
1021 * written by write(2) is visible in mmap.
1022 */
1023 if ((iomap->flags & IOMAP_F_NEW) && inode->i_mapping->nrpages) {
1024 invalidate_inode_pages2_range(inode->i_mapping,
1025 pos >> PAGE_SHIFT,
1026 (end - 1) >> PAGE_SHIFT);
1027 }
1028
Christoph Hellwiga254e562016-09-19 11:24:49 +10001029 while (pos < end) {
1030 unsigned offset = pos & (PAGE_SIZE - 1);
1031 struct blk_dax_ctl dax = { 0 };
1032 ssize_t map_len;
1033
Ross Zwisler333ccc92016-11-08 11:33:09 +11001034 dax.sector = dax_iomap_sector(iomap, pos);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001035 dax.size = (length + offset + PAGE_SIZE - 1) & PAGE_MASK;
1036 map_len = dax_map_atomic(iomap->bdev, &dax);
1037 if (map_len < 0) {
1038 ret = map_len;
1039 break;
1040 }
1041
1042 dax.addr += offset;
1043 map_len -= offset;
1044 if (map_len > end - pos)
1045 map_len = end - pos;
1046
1047 if (iov_iter_rw(iter) == WRITE)
1048 map_len = copy_from_iter_pmem(dax.addr, map_len, iter);
1049 else
1050 map_len = copy_to_iter(dax.addr, map_len, iter);
1051 dax_unmap_atomic(iomap->bdev, &dax);
1052 if (map_len <= 0) {
1053 ret = map_len ? map_len : -EFAULT;
1054 break;
1055 }
1056
1057 pos += map_len;
1058 length -= map_len;
1059 done += map_len;
1060 }
1061
1062 return done ? done : ret;
1063}
1064
1065/**
Ross Zwisler11c59c92016-11-08 11:32:46 +11001066 * dax_iomap_rw - Perform I/O to a DAX file
Christoph Hellwiga254e562016-09-19 11:24:49 +10001067 * @iocb: The control block for this I/O
1068 * @iter: The addresses to do I/O from or to
1069 * @ops: iomap ops passed from the file system
1070 *
1071 * This function performs read and write operations to directly mapped
1072 * persistent memory. The callers needs to take care of read/write exclusion
1073 * and evicting any page cache pages in the region under I/O.
1074 */
1075ssize_t
Ross Zwisler11c59c92016-11-08 11:32:46 +11001076dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
Christoph Hellwiga254e562016-09-19 11:24:49 +10001077 struct iomap_ops *ops)
1078{
1079 struct address_space *mapping = iocb->ki_filp->f_mapping;
1080 struct inode *inode = mapping->host;
1081 loff_t pos = iocb->ki_pos, ret = 0, done = 0;
1082 unsigned flags = 0;
1083
1084 if (iov_iter_rw(iter) == WRITE)
1085 flags |= IOMAP_WRITE;
1086
Christoph Hellwiga254e562016-09-19 11:24:49 +10001087 while (iov_iter_count(iter)) {
1088 ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
Ross Zwisler11c59c92016-11-08 11:32:46 +11001089 iter, dax_iomap_actor);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001090 if (ret <= 0)
1091 break;
1092 pos += ret;
1093 done += ret;
1094 }
1095
1096 iocb->ki_pos += done;
1097 return done ? done : ret;
1098}
Ross Zwisler11c59c92016-11-08 11:32:46 +11001099EXPORT_SYMBOL_GPL(dax_iomap_rw);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001100
Jan Kara9f141d62016-10-19 14:34:31 +02001101static int dax_fault_return(int error)
1102{
1103 if (error == 0)
1104 return VM_FAULT_NOPAGE;
1105 if (error == -ENOMEM)
1106 return VM_FAULT_OOM;
1107 return VM_FAULT_SIGBUS;
1108}
1109
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001110/**
Ross Zwisler11c59c92016-11-08 11:32:46 +11001111 * dax_iomap_fault - handle a page fault on a DAX file
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001112 * @vma: The virtual memory area where the fault occurred
1113 * @vmf: The description of the fault
1114 * @ops: iomap ops passed from the file system
1115 *
1116 * When a page fault occurs, filesystems may call this helper in their fault
1117 * or mkwrite handler for DAX files. Assumes the caller has done all the
1118 * necessary locking for the page fault to proceed successfully.
1119 */
Ross Zwisler11c59c92016-11-08 11:32:46 +11001120int dax_iomap_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001121 struct iomap_ops *ops)
1122{
1123 struct address_space *mapping = vma->vm_file->f_mapping;
1124 struct inode *inode = mapping->host;
Jan Kara1a29d852016-12-14 15:07:01 -08001125 unsigned long vaddr = vmf->address;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001126 loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
1127 sector_t sector;
1128 struct iomap iomap = { 0 };
Jan Kara9484ab12016-11-10 10:26:50 +11001129 unsigned flags = IOMAP_FAULT;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001130 int error, major = 0;
Jan Karab1aa8122016-12-14 15:07:24 -08001131 int vmf_ret = 0;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001132 void *entry;
1133
1134 /*
1135 * Check whether offset isn't beyond end of file now. Caller is supposed
1136 * to hold locks serializing us with truncate / punch hole so this is
1137 * a reliable test.
1138 */
1139 if (pos >= i_size_read(inode))
1140 return VM_FAULT_SIGBUS;
1141
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001142 if ((vmf->flags & FAULT_FLAG_WRITE) && !vmf->cow_page)
1143 flags |= IOMAP_WRITE;
1144
1145 /*
1146 * Note that we don't bother to use iomap_apply here: DAX required
1147 * the file system block size to be equal the page size, which means
1148 * that we never have to deal with more than a single extent here.
1149 */
1150 error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap);
1151 if (error)
Jan Kara9f141d62016-10-19 14:34:31 +02001152 return dax_fault_return(error);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001153 if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
Jan Kara9f141d62016-10-19 14:34:31 +02001154 vmf_ret = dax_fault_return(-EIO); /* fs corruption? */
1155 goto finish_iomap;
1156 }
1157
1158 entry = grab_mapping_entry(mapping, vmf->pgoff, 0);
1159 if (IS_ERR(entry)) {
1160 vmf_ret = dax_fault_return(PTR_ERR(entry));
Ross Zwisler15502902016-11-08 11:33:26 +11001161 goto finish_iomap;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001162 }
1163
Ross Zwisler333ccc92016-11-08 11:33:09 +11001164 sector = dax_iomap_sector(&iomap, pos);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001165
1166 if (vmf->cow_page) {
1167 switch (iomap.type) {
1168 case IOMAP_HOLE:
1169 case IOMAP_UNWRITTEN:
1170 clear_user_highpage(vmf->cow_page, vaddr);
1171 break;
1172 case IOMAP_MAPPED:
1173 error = copy_user_dax(iomap.bdev, sector, PAGE_SIZE,
1174 vmf->cow_page, vaddr);
1175 break;
1176 default:
1177 WARN_ON_ONCE(1);
1178 error = -EIO;
1179 break;
1180 }
1181
1182 if (error)
Jan Kara9f141d62016-10-19 14:34:31 +02001183 goto error_unlock_entry;
Jan Karab1aa8122016-12-14 15:07:24 -08001184
1185 __SetPageUptodate(vmf->cow_page);
1186 vmf_ret = finish_fault(vmf);
1187 if (!vmf_ret)
1188 vmf_ret = VM_FAULT_DONE_COW;
Jan Kara9f141d62016-10-19 14:34:31 +02001189 goto unlock_entry;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001190 }
1191
1192 switch (iomap.type) {
1193 case IOMAP_MAPPED:
1194 if (iomap.flags & IOMAP_F_NEW) {
1195 count_vm_event(PGMAJFAULT);
1196 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
1197 major = VM_FAULT_MAJOR;
1198 }
1199 error = dax_insert_mapping(mapping, iomap.bdev, sector,
1200 PAGE_SIZE, &entry, vma, vmf);
Jan Kara9f141d62016-10-19 14:34:31 +02001201 /* -EBUSY is fine, somebody else faulted on the same PTE */
1202 if (error == -EBUSY)
1203 error = 0;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001204 break;
1205 case IOMAP_UNWRITTEN:
1206 case IOMAP_HOLE:
Ross Zwisler15502902016-11-08 11:33:26 +11001207 if (!(vmf->flags & FAULT_FLAG_WRITE)) {
Jan Karaf449b932016-10-19 14:48:38 +02001208 vmf_ret = dax_load_hole(mapping, &entry, vmf);
Jan Kara9f141d62016-10-19 14:34:31 +02001209 goto unlock_entry;
Ross Zwisler15502902016-11-08 11:33:26 +11001210 }
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001211 /*FALLTHRU*/
1212 default:
1213 WARN_ON_ONCE(1);
1214 error = -EIO;
1215 break;
1216 }
1217
Jan Kara9f141d62016-10-19 14:34:31 +02001218 error_unlock_entry:
1219 vmf_ret = dax_fault_return(error) | major;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001220 unlock_entry:
Jan Karaf449b932016-10-19 14:48:38 +02001221 put_locked_mapping_entry(mapping, vmf->pgoff, entry);
Jan Kara9f141d62016-10-19 14:34:31 +02001222 finish_iomap:
1223 if (ops->iomap_end) {
1224 int copied = PAGE_SIZE;
1225
1226 if (vmf_ret & VM_FAULT_ERROR)
1227 copied = 0;
1228 /*
1229 * The fault is done by now and there's no way back (other
1230 * thread may be already happily using PTE we have installed).
1231 * Just ignore error from ->iomap_end since we cannot do much
1232 * with it.
1233 */
1234 ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap);
Ross Zwisler15502902016-11-08 11:33:26 +11001235 }
Jan Kara9f141d62016-10-19 14:34:31 +02001236 return vmf_ret;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001237}
Ross Zwisler11c59c92016-11-08 11:32:46 +11001238EXPORT_SYMBOL_GPL(dax_iomap_fault);
Ross Zwisler642261a2016-11-08 11:34:45 +11001239
1240#ifdef CONFIG_FS_DAX_PMD
1241/*
1242 * The 'colour' (ie low bits) within a PMD of a page offset. This comes up
1243 * more often than one might expect in the below functions.
1244 */
1245#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1)
1246
1247static int dax_pmd_insert_mapping(struct vm_area_struct *vma, pmd_t *pmd,
1248 struct vm_fault *vmf, unsigned long address,
1249 struct iomap *iomap, loff_t pos, bool write, void **entryp)
1250{
1251 struct address_space *mapping = vma->vm_file->f_mapping;
1252 struct block_device *bdev = iomap->bdev;
1253 struct blk_dax_ctl dax = {
1254 .sector = dax_iomap_sector(iomap, pos),
1255 .size = PMD_SIZE,
1256 };
1257 long length = dax_map_atomic(bdev, &dax);
1258 void *ret;
1259
1260 if (length < 0) /* dax_map_atomic() failed */
1261 return VM_FAULT_FALLBACK;
1262 if (length < PMD_SIZE)
1263 goto unmap_fallback;
1264 if (pfn_t_to_pfn(dax.pfn) & PG_PMD_COLOUR)
1265 goto unmap_fallback;
1266 if (!pfn_t_devmap(dax.pfn))
1267 goto unmap_fallback;
1268
1269 dax_unmap_atomic(bdev, &dax);
1270
1271 ret = dax_insert_mapping_entry(mapping, vmf, *entryp, dax.sector,
1272 RADIX_DAX_PMD);
1273 if (IS_ERR(ret))
1274 return VM_FAULT_FALLBACK;
1275 *entryp = ret;
1276
1277 return vmf_insert_pfn_pmd(vma, address, pmd, dax.pfn, write);
1278
1279 unmap_fallback:
1280 dax_unmap_atomic(bdev, &dax);
1281 return VM_FAULT_FALLBACK;
1282}
1283
1284static int dax_pmd_load_hole(struct vm_area_struct *vma, pmd_t *pmd,
1285 struct vm_fault *vmf, unsigned long address,
1286 struct iomap *iomap, void **entryp)
1287{
1288 struct address_space *mapping = vma->vm_file->f_mapping;
1289 unsigned long pmd_addr = address & PMD_MASK;
1290 struct page *zero_page;
1291 spinlock_t *ptl;
1292 pmd_t pmd_entry;
1293 void *ret;
1294
1295 zero_page = mm_get_huge_zero_page(vma->vm_mm);
1296
1297 if (unlikely(!zero_page))
1298 return VM_FAULT_FALLBACK;
1299
1300 ret = dax_insert_mapping_entry(mapping, vmf, *entryp, 0,
1301 RADIX_DAX_PMD | RADIX_DAX_HZP);
1302 if (IS_ERR(ret))
1303 return VM_FAULT_FALLBACK;
1304 *entryp = ret;
1305
1306 ptl = pmd_lock(vma->vm_mm, pmd);
1307 if (!pmd_none(*pmd)) {
1308 spin_unlock(ptl);
1309 return VM_FAULT_FALLBACK;
1310 }
1311
1312 pmd_entry = mk_pmd(zero_page, vma->vm_page_prot);
1313 pmd_entry = pmd_mkhuge(pmd_entry);
1314 set_pmd_at(vma->vm_mm, pmd_addr, pmd, pmd_entry);
1315 spin_unlock(ptl);
1316 return VM_FAULT_NOPAGE;
1317}
1318
1319int dax_iomap_pmd_fault(struct vm_area_struct *vma, unsigned long address,
1320 pmd_t *pmd, unsigned int flags, struct iomap_ops *ops)
1321{
1322 struct address_space *mapping = vma->vm_file->f_mapping;
1323 unsigned long pmd_addr = address & PMD_MASK;
1324 bool write = flags & FAULT_FLAG_WRITE;
Jan Kara9484ab12016-11-10 10:26:50 +11001325 unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;
Ross Zwisler642261a2016-11-08 11:34:45 +11001326 struct inode *inode = mapping->host;
1327 int result = VM_FAULT_FALLBACK;
1328 struct iomap iomap = { 0 };
1329 pgoff_t max_pgoff, pgoff;
1330 struct vm_fault vmf;
1331 void *entry;
1332 loff_t pos;
1333 int error;
1334
1335 /* Fall back to PTEs if we're going to COW */
1336 if (write && !(vma->vm_flags & VM_SHARED))
1337 goto fallback;
1338
1339 /* If the PMD would extend outside the VMA */
1340 if (pmd_addr < vma->vm_start)
1341 goto fallback;
1342 if ((pmd_addr + PMD_SIZE) > vma->vm_end)
1343 goto fallback;
1344
1345 /*
1346 * Check whether offset isn't beyond end of file now. Caller is
1347 * supposed to hold locks serializing us with truncate / punch hole so
1348 * this is a reliable test.
1349 */
1350 pgoff = linear_page_index(vma, pmd_addr);
1351 max_pgoff = (i_size_read(inode) - 1) >> PAGE_SHIFT;
1352
1353 if (pgoff > max_pgoff)
1354 return VM_FAULT_SIGBUS;
1355
1356 /* If the PMD would extend beyond the file size */
1357 if ((pgoff | PG_PMD_COLOUR) > max_pgoff)
1358 goto fallback;
1359
1360 /*
Ross Zwisler642261a2016-11-08 11:34:45 +11001361 * Note that we don't use iomap_apply here. We aren't doing I/O, only
1362 * setting up a mapping, so really we're using iomap_begin() as a way
1363 * to look up our filesystem block.
1364 */
1365 pos = (loff_t)pgoff << PAGE_SHIFT;
1366 error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap);
1367 if (error)
Jan Kara9f141d62016-10-19 14:34:31 +02001368 goto fallback;
1369
Ross Zwisler642261a2016-11-08 11:34:45 +11001370 if (iomap.offset + iomap.length < pos + PMD_SIZE)
1371 goto finish_iomap;
1372
Jan Kara9f141d62016-10-19 14:34:31 +02001373 /*
1374 * grab_mapping_entry() will make sure we get a 2M empty entry, a DAX
1375 * PMD or a HZP entry. If it can't (because a 4k page is already in
1376 * the tree, for instance), it will return -EEXIST and we just fall
1377 * back to 4k entries.
1378 */
1379 entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD);
1380 if (IS_ERR(entry))
1381 goto finish_iomap;
1382
Ross Zwisler642261a2016-11-08 11:34:45 +11001383 vmf.pgoff = pgoff;
1384 vmf.flags = flags;
1385 vmf.gfp_mask = mapping_gfp_mask(mapping) | __GFP_IO;
1386
1387 switch (iomap.type) {
1388 case IOMAP_MAPPED:
1389 result = dax_pmd_insert_mapping(vma, pmd, &vmf, address,
1390 &iomap, pos, write, &entry);
1391 break;
1392 case IOMAP_UNWRITTEN:
1393 case IOMAP_HOLE:
1394 if (WARN_ON_ONCE(write))
Jan Kara9f141d62016-10-19 14:34:31 +02001395 goto unlock_entry;
Ross Zwisler642261a2016-11-08 11:34:45 +11001396 result = dax_pmd_load_hole(vma, pmd, &vmf, address, &iomap,
1397 &entry);
1398 break;
1399 default:
1400 WARN_ON_ONCE(1);
1401 break;
1402 }
1403
Ross Zwisler642261a2016-11-08 11:34:45 +11001404 unlock_entry:
1405 put_locked_mapping_entry(mapping, pgoff, entry);
Jan Kara9f141d62016-10-19 14:34:31 +02001406 finish_iomap:
1407 if (ops->iomap_end) {
1408 int copied = PMD_SIZE;
1409
1410 if (result == VM_FAULT_FALLBACK)
1411 copied = 0;
1412 /*
1413 * The fault is done by now and there's no way back (other
1414 * thread may be already happily using PMD we have installed).
1415 * Just ignore error from ->iomap_end since we cannot do much
1416 * with it.
1417 */
1418 ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags,
1419 &iomap);
1420 }
Ross Zwisler642261a2016-11-08 11:34:45 +11001421 fallback:
1422 if (result == VM_FAULT_FALLBACK) {
1423 split_huge_pmd(vma, pmd, address);
1424 count_vm_event(THP_FAULT_FALLBACK);
1425 }
1426 return result;
1427}
1428EXPORT_SYMBOL_GPL(dax_iomap_pmd_fault);
1429#endif /* CONFIG_FS_DAX_PMD */