blob: 5c74f60d0a5094dc0a27f27ae0acd41667414332 [file] [log] [blame]
Matthew Wilcoxd475c632015-02-16 15:58:56 -08001/*
2 * fs/dax.c - Direct Access filesystem code
3 * Copyright (c) 2013-2014 Intel Corporation
4 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
5 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 */
16
17#include <linux/atomic.h>
18#include <linux/blkdev.h>
19#include <linux/buffer_head.h>
Ross Zwislerd77e92e2015-09-09 10:29:40 -060020#include <linux/dax.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080021#include <linux/fs.h>
22#include <linux/genhd.h>
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -080023#include <linux/highmem.h>
24#include <linux/memcontrol.h>
25#include <linux/mm.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080026#include <linux/mutex.h>
Ross Zwisler9973c982016-01-22 15:10:47 -080027#include <linux/pagevec.h>
Ross Zwisler2765cfb2015-08-18 13:55:40 -060028#include <linux/pmem.h>
Matthew Wilcox289c6ae2015-02-16 15:58:59 -080029#include <linux/sched.h>
Matthew Wilcoxd475c632015-02-16 15:58:56 -080030#include <linux/uio.h>
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -080031#include <linux/vmstat.h>
Dan Williams34c0fd52016-01-15 16:56:14 -080032#include <linux/pfn_t.h>
Dan Williams0e749e52016-01-15 16:55:53 -080033#include <linux/sizes.h>
Jan Kara4b4bb462016-12-14 15:07:53 -080034#include <linux/mmu_notifier.h>
Christoph Hellwiga254e562016-09-19 11:24:49 +100035#include <linux/iomap.h>
36#include "internal.h"
Matthew Wilcoxd475c632015-02-16 15:58:56 -080037
Jan Karaac401cc2016-05-12 18:29:18 +020038/* We choose 4096 entries - same as per-zone page wait tables */
39#define DAX_WAIT_TABLE_BITS 12
40#define DAX_WAIT_TABLE_ENTRIES (1 << DAX_WAIT_TABLE_BITS)
41
Ross Zwislerce95ab02016-11-08 11:31:44 +110042static wait_queue_head_t wait_table[DAX_WAIT_TABLE_ENTRIES];
Jan Karaac401cc2016-05-12 18:29:18 +020043
44static int __init init_dax_wait_table(void)
45{
46 int i;
47
48 for (i = 0; i < DAX_WAIT_TABLE_ENTRIES; i++)
49 init_waitqueue_head(wait_table + i);
50 return 0;
51}
52fs_initcall(init_dax_wait_table);
53
Dan Williamsb2e0d162016-01-15 16:55:59 -080054static long dax_map_atomic(struct block_device *bdev, struct blk_dax_ctl *dax)
55{
56 struct request_queue *q = bdev->bd_queue;
57 long rc = -EIO;
58
Dan Williams7a9eb202016-06-03 18:06:47 -070059 dax->addr = ERR_PTR(-EIO);
Dan Williamsb2e0d162016-01-15 16:55:59 -080060 if (blk_queue_enter(q, true) != 0)
61 return rc;
62
63 rc = bdev_direct_access(bdev, dax);
64 if (rc < 0) {
Dan Williams7a9eb202016-06-03 18:06:47 -070065 dax->addr = ERR_PTR(rc);
Dan Williamsb2e0d162016-01-15 16:55:59 -080066 blk_queue_exit(q);
67 return rc;
68 }
69 return rc;
70}
71
72static void dax_unmap_atomic(struct block_device *bdev,
73 const struct blk_dax_ctl *dax)
74{
75 if (IS_ERR(dax->addr))
76 return;
77 blk_queue_exit(bdev->bd_queue);
78}
79
Ross Zwisler642261a2016-11-08 11:34:45 +110080static int dax_is_pmd_entry(void *entry)
81{
82 return (unsigned long)entry & RADIX_DAX_PMD;
83}
84
85static int dax_is_pte_entry(void *entry)
86{
87 return !((unsigned long)entry & RADIX_DAX_PMD);
88}
89
90static int dax_is_zero_entry(void *entry)
91{
92 return (unsigned long)entry & RADIX_DAX_HZP;
93}
94
95static int dax_is_empty_entry(void *entry)
96{
97 return (unsigned long)entry & RADIX_DAX_EMPTY;
98}
99
Dan Williamsd1a5f2b42016-01-28 20:25:31 -0800100struct page *read_dax_sector(struct block_device *bdev, sector_t n)
101{
102 struct page *page = alloc_pages(GFP_KERNEL, 0);
103 struct blk_dax_ctl dax = {
104 .size = PAGE_SIZE,
105 .sector = n & ~((((int) PAGE_SIZE) / 512) - 1),
106 };
107 long rc;
108
109 if (!page)
110 return ERR_PTR(-ENOMEM);
111
112 rc = dax_map_atomic(bdev, &dax);
113 if (rc < 0)
114 return ERR_PTR(rc);
115 memcpy_from_pmem(page_address(page), dax.addr, PAGE_SIZE);
116 dax_unmap_atomic(bdev, &dax);
117 return page;
118}
119
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800120/*
Jan Karaac401cc2016-05-12 18:29:18 +0200121 * DAX radix tree locking
122 */
123struct exceptional_entry_key {
124 struct address_space *mapping;
Ross Zwisler63e95b52016-11-08 11:32:20 +1100125 pgoff_t entry_start;
Jan Karaac401cc2016-05-12 18:29:18 +0200126};
127
128struct wait_exceptional_entry_queue {
129 wait_queue_t wait;
130 struct exceptional_entry_key key;
131};
132
Ross Zwisler63e95b52016-11-08 11:32:20 +1100133static wait_queue_head_t *dax_entry_waitqueue(struct address_space *mapping,
134 pgoff_t index, void *entry, struct exceptional_entry_key *key)
135{
136 unsigned long hash;
137
138 /*
139 * If 'entry' is a PMD, align the 'index' that we use for the wait
140 * queue to the start of that PMD. This ensures that all offsets in
141 * the range covered by the PMD map to the same bit lock.
142 */
Ross Zwisler642261a2016-11-08 11:34:45 +1100143 if (dax_is_pmd_entry(entry))
Ross Zwisler63e95b52016-11-08 11:32:20 +1100144 index &= ~((1UL << (PMD_SHIFT - PAGE_SHIFT)) - 1);
145
146 key->mapping = mapping;
147 key->entry_start = index;
148
149 hash = hash_long((unsigned long)mapping ^ index, DAX_WAIT_TABLE_BITS);
150 return wait_table + hash;
151}
152
Jan Karaac401cc2016-05-12 18:29:18 +0200153static int wake_exceptional_entry_func(wait_queue_t *wait, unsigned int mode,
154 int sync, void *keyp)
155{
156 struct exceptional_entry_key *key = keyp;
157 struct wait_exceptional_entry_queue *ewait =
158 container_of(wait, struct wait_exceptional_entry_queue, wait);
159
160 if (key->mapping != ewait->key.mapping ||
Ross Zwisler63e95b52016-11-08 11:32:20 +1100161 key->entry_start != ewait->key.entry_start)
Jan Karaac401cc2016-05-12 18:29:18 +0200162 return 0;
163 return autoremove_wake_function(wait, mode, sync, NULL);
164}
165
166/*
167 * Check whether the given slot is locked. The function must be called with
168 * mapping->tree_lock held
169 */
170static inline int slot_locked(struct address_space *mapping, void **slot)
171{
172 unsigned long entry = (unsigned long)
173 radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
174 return entry & RADIX_DAX_ENTRY_LOCK;
175}
176
177/*
178 * Mark the given slot is locked. The function must be called with
179 * mapping->tree_lock held
180 */
181static inline void *lock_slot(struct address_space *mapping, void **slot)
182{
183 unsigned long entry = (unsigned long)
184 radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
185
186 entry |= RADIX_DAX_ENTRY_LOCK;
Johannes Weiner6d75f362016-12-12 16:43:43 -0800187 radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry);
Jan Karaac401cc2016-05-12 18:29:18 +0200188 return (void *)entry;
189}
190
191/*
192 * Mark the given slot is unlocked. The function must be called with
193 * mapping->tree_lock held
194 */
195static inline void *unlock_slot(struct address_space *mapping, void **slot)
196{
197 unsigned long entry = (unsigned long)
198 radix_tree_deref_slot_protected(slot, &mapping->tree_lock);
199
200 entry &= ~(unsigned long)RADIX_DAX_ENTRY_LOCK;
Johannes Weiner6d75f362016-12-12 16:43:43 -0800201 radix_tree_replace_slot(&mapping->page_tree, slot, (void *)entry);
Jan Karaac401cc2016-05-12 18:29:18 +0200202 return (void *)entry;
203}
204
205/*
206 * Lookup entry in radix tree, wait for it to become unlocked if it is
207 * exceptional entry and return it. The caller must call
208 * put_unlocked_mapping_entry() when he decided not to lock the entry or
209 * put_locked_mapping_entry() when he locked the entry and now wants to
210 * unlock it.
211 *
212 * The function must be called with mapping->tree_lock held.
213 */
214static void *get_unlocked_mapping_entry(struct address_space *mapping,
215 pgoff_t index, void ***slotp)
216{
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100217 void *entry, **slot;
Jan Karaac401cc2016-05-12 18:29:18 +0200218 struct wait_exceptional_entry_queue ewait;
Ross Zwisler63e95b52016-11-08 11:32:20 +1100219 wait_queue_head_t *wq;
Jan Karaac401cc2016-05-12 18:29:18 +0200220
221 init_wait(&ewait.wait);
222 ewait.wait.func = wake_exceptional_entry_func;
Jan Karaac401cc2016-05-12 18:29:18 +0200223
224 for (;;) {
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100225 entry = __radix_tree_lookup(&mapping->page_tree, index, NULL,
Jan Karaac401cc2016-05-12 18:29:18 +0200226 &slot);
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100227 if (!entry || !radix_tree_exceptional_entry(entry) ||
Jan Karaac401cc2016-05-12 18:29:18 +0200228 !slot_locked(mapping, slot)) {
229 if (slotp)
230 *slotp = slot;
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100231 return entry;
Jan Karaac401cc2016-05-12 18:29:18 +0200232 }
Ross Zwisler63e95b52016-11-08 11:32:20 +1100233
234 wq = dax_entry_waitqueue(mapping, index, entry, &ewait.key);
Jan Karaac401cc2016-05-12 18:29:18 +0200235 prepare_to_wait_exclusive(wq, &ewait.wait,
236 TASK_UNINTERRUPTIBLE);
237 spin_unlock_irq(&mapping->tree_lock);
238 schedule();
239 finish_wait(wq, &ewait.wait);
240 spin_lock_irq(&mapping->tree_lock);
241 }
242}
243
Jan Karab1aa8122016-12-14 15:07:24 -0800244static void dax_unlock_mapping_entry(struct address_space *mapping,
245 pgoff_t index)
246{
247 void *entry, **slot;
248
249 spin_lock_irq(&mapping->tree_lock);
250 entry = __radix_tree_lookup(&mapping->page_tree, index, NULL, &slot);
251 if (WARN_ON_ONCE(!entry || !radix_tree_exceptional_entry(entry) ||
252 !slot_locked(mapping, slot))) {
253 spin_unlock_irq(&mapping->tree_lock);
254 return;
255 }
256 unlock_slot(mapping, slot);
257 spin_unlock_irq(&mapping->tree_lock);
258 dax_wake_mapping_entry_waiter(mapping, index, entry, false);
259}
260
Jan Karaac401cc2016-05-12 18:29:18 +0200261static void put_locked_mapping_entry(struct address_space *mapping,
262 pgoff_t index, void *entry)
263{
264 if (!radix_tree_exceptional_entry(entry)) {
265 unlock_page(entry);
266 put_page(entry);
267 } else {
Jan Karabc2466e2016-05-12 18:29:19 +0200268 dax_unlock_mapping_entry(mapping, index);
Jan Karaac401cc2016-05-12 18:29:18 +0200269 }
270}
271
272/*
273 * Called when we are done with radix tree entry we looked up via
274 * get_unlocked_mapping_entry() and which we didn't lock in the end.
275 */
276static void put_unlocked_mapping_entry(struct address_space *mapping,
277 pgoff_t index, void *entry)
278{
279 if (!radix_tree_exceptional_entry(entry))
280 return;
281
282 /* We have to wake up next waiter for the radix tree entry lock */
Ross Zwisler422476c2016-11-08 11:33:44 +1100283 dax_wake_mapping_entry_waiter(mapping, index, entry, false);
284}
285
Jan Karaac401cc2016-05-12 18:29:18 +0200286/*
287 * Find radix tree entry at given index. If it points to a page, return with
288 * the page locked. If it points to the exceptional entry, return with the
289 * radix tree entry locked. If the radix tree doesn't contain given index,
290 * create empty exceptional entry for the index and return with it locked.
291 *
Ross Zwisler642261a2016-11-08 11:34:45 +1100292 * When requesting an entry with size RADIX_DAX_PMD, grab_mapping_entry() will
293 * either return that locked entry or will return an error. This error will
294 * happen if there are any 4k entries (either zero pages or DAX entries)
295 * within the 2MiB range that we are requesting.
296 *
297 * We always favor 4k entries over 2MiB entries. There isn't a flow where we
298 * evict 4k entries in order to 'upgrade' them to a 2MiB entry. A 2MiB
299 * insertion will fail if it finds any 4k entries already in the tree, and a
300 * 4k insertion will cause an existing 2MiB entry to be unmapped and
301 * downgraded to 4k entries. This happens for both 2MiB huge zero pages as
302 * well as 2MiB empty entries.
303 *
304 * The exception to this downgrade path is for 2MiB DAX PMD entries that have
305 * real storage backing them. We will leave these real 2MiB DAX entries in
306 * the tree, and PTE writes will simply dirty the entire 2MiB DAX entry.
307 *
Jan Karaac401cc2016-05-12 18:29:18 +0200308 * Note: Unlike filemap_fault() we don't honor FAULT_FLAG_RETRY flags. For
309 * persistent memory the benefit is doubtful. We can add that later if we can
310 * show it helps.
311 */
Ross Zwisler642261a2016-11-08 11:34:45 +1100312static void *grab_mapping_entry(struct address_space *mapping, pgoff_t index,
313 unsigned long size_flag)
Jan Karaac401cc2016-05-12 18:29:18 +0200314{
Ross Zwisler642261a2016-11-08 11:34:45 +1100315 bool pmd_downgrade = false; /* splitting 2MiB entry into 4k entries? */
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100316 void *entry, **slot;
Jan Karaac401cc2016-05-12 18:29:18 +0200317
318restart:
319 spin_lock_irq(&mapping->tree_lock);
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100320 entry = get_unlocked_mapping_entry(mapping, index, &slot);
Ross Zwisler642261a2016-11-08 11:34:45 +1100321
322 if (entry) {
323 if (size_flag & RADIX_DAX_PMD) {
324 if (!radix_tree_exceptional_entry(entry) ||
325 dax_is_pte_entry(entry)) {
326 put_unlocked_mapping_entry(mapping, index,
327 entry);
328 entry = ERR_PTR(-EEXIST);
329 goto out_unlock;
330 }
331 } else { /* trying to grab a PTE entry */
332 if (radix_tree_exceptional_entry(entry) &&
333 dax_is_pmd_entry(entry) &&
334 (dax_is_zero_entry(entry) ||
335 dax_is_empty_entry(entry))) {
336 pmd_downgrade = true;
337 }
338 }
339 }
340
Jan Karaac401cc2016-05-12 18:29:18 +0200341 /* No entry for given index? Make sure radix tree is big enough. */
Ross Zwisler642261a2016-11-08 11:34:45 +1100342 if (!entry || pmd_downgrade) {
Jan Karaac401cc2016-05-12 18:29:18 +0200343 int err;
344
Ross Zwisler642261a2016-11-08 11:34:45 +1100345 if (pmd_downgrade) {
346 /*
347 * Make sure 'entry' remains valid while we drop
348 * mapping->tree_lock.
349 */
350 entry = lock_slot(mapping, slot);
351 }
352
Jan Karaac401cc2016-05-12 18:29:18 +0200353 spin_unlock_irq(&mapping->tree_lock);
Ross Zwisler642261a2016-11-08 11:34:45 +1100354 /*
355 * Besides huge zero pages the only other thing that gets
356 * downgraded are empty entries which don't need to be
357 * unmapped.
358 */
359 if (pmd_downgrade && dax_is_zero_entry(entry))
360 unmap_mapping_range(mapping,
361 (index << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0);
362
Jan Kara0cb80b42016-12-12 21:34:12 -0500363 err = radix_tree_preload(
364 mapping_gfp_mask(mapping) & ~__GFP_HIGHMEM);
365 if (err) {
366 if (pmd_downgrade)
367 put_locked_mapping_entry(mapping, index, entry);
368 return ERR_PTR(err);
369 }
Jan Karaac401cc2016-05-12 18:29:18 +0200370 spin_lock_irq(&mapping->tree_lock);
Ross Zwisler642261a2016-11-08 11:34:45 +1100371
372 if (pmd_downgrade) {
373 radix_tree_delete(&mapping->page_tree, index);
374 mapping->nrexceptional--;
375 dax_wake_mapping_entry_waiter(mapping, index, entry,
376 true);
377 }
378
379 entry = dax_radix_locked_entry(0, size_flag | RADIX_DAX_EMPTY);
380
381 err = __radix_tree_insert(&mapping->page_tree, index,
382 dax_radix_order(entry), entry);
Jan Karaac401cc2016-05-12 18:29:18 +0200383 radix_tree_preload_end();
384 if (err) {
385 spin_unlock_irq(&mapping->tree_lock);
Ross Zwisler642261a2016-11-08 11:34:45 +1100386 /*
387 * Someone already created the entry? This is a
388 * normal failure when inserting PMDs in a range
389 * that already contains PTEs. In that case we want
390 * to return -EEXIST immediately.
391 */
392 if (err == -EEXIST && !(size_flag & RADIX_DAX_PMD))
Jan Karaac401cc2016-05-12 18:29:18 +0200393 goto restart;
Ross Zwisler642261a2016-11-08 11:34:45 +1100394 /*
395 * Our insertion of a DAX PMD entry failed, most
396 * likely because it collided with a PTE sized entry
397 * at a different index in the PMD range. We haven't
398 * inserted anything into the radix tree and have no
399 * waiters to wake.
400 */
Jan Karaac401cc2016-05-12 18:29:18 +0200401 return ERR_PTR(err);
402 }
403 /* Good, we have inserted empty locked entry into the tree. */
404 mapping->nrexceptional++;
405 spin_unlock_irq(&mapping->tree_lock);
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100406 return entry;
Jan Karaac401cc2016-05-12 18:29:18 +0200407 }
408 /* Normal page in radix tree? */
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100409 if (!radix_tree_exceptional_entry(entry)) {
410 struct page *page = entry;
Jan Karaac401cc2016-05-12 18:29:18 +0200411
412 get_page(page);
413 spin_unlock_irq(&mapping->tree_lock);
414 lock_page(page);
415 /* Page got truncated? Retry... */
416 if (unlikely(page->mapping != mapping)) {
417 unlock_page(page);
418 put_page(page);
419 goto restart;
420 }
421 return page;
422 }
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100423 entry = lock_slot(mapping, slot);
Ross Zwisler642261a2016-11-08 11:34:45 +1100424 out_unlock:
Jan Karaac401cc2016-05-12 18:29:18 +0200425 spin_unlock_irq(&mapping->tree_lock);
Ross Zwislere3ad61c2016-11-08 11:32:12 +1100426 return entry;
Jan Karaac401cc2016-05-12 18:29:18 +0200427}
428
Ross Zwisler63e95b52016-11-08 11:32:20 +1100429/*
430 * We do not necessarily hold the mapping->tree_lock when we call this
431 * function so it is possible that 'entry' is no longer a valid item in the
Ross Zwisler642261a2016-11-08 11:34:45 +1100432 * radix tree. This is okay because all we really need to do is to find the
433 * correct waitqueue where tasks might be waiting for that old 'entry' and
434 * wake them.
Ross Zwisler63e95b52016-11-08 11:32:20 +1100435 */
Jan Karaac401cc2016-05-12 18:29:18 +0200436void dax_wake_mapping_entry_waiter(struct address_space *mapping,
Ross Zwisler63e95b52016-11-08 11:32:20 +1100437 pgoff_t index, void *entry, bool wake_all)
Jan Karaac401cc2016-05-12 18:29:18 +0200438{
Ross Zwisler63e95b52016-11-08 11:32:20 +1100439 struct exceptional_entry_key key;
440 wait_queue_head_t *wq;
441
442 wq = dax_entry_waitqueue(mapping, index, entry, &key);
Jan Karaac401cc2016-05-12 18:29:18 +0200443
444 /*
445 * Checking for locked entry and prepare_to_wait_exclusive() happens
446 * under mapping->tree_lock, ditto for entry handling in our callers.
447 * So at this point all tasks that could have seen our entry locked
448 * must be in the waitqueue and the following check will see them.
449 */
Ross Zwisler63e95b52016-11-08 11:32:20 +1100450 if (waitqueue_active(wq))
Jan Karaac401cc2016-05-12 18:29:18 +0200451 __wake_up(wq, TASK_NORMAL, wake_all ? 0 : 1, &key);
Jan Karaac401cc2016-05-12 18:29:18 +0200452}
453
Jan Karac6dcf522016-08-10 17:22:44 +0200454static int __dax_invalidate_mapping_entry(struct address_space *mapping,
455 pgoff_t index, bool trunc)
456{
457 int ret = 0;
458 void *entry;
459 struct radix_tree_root *page_tree = &mapping->page_tree;
460
461 spin_lock_irq(&mapping->tree_lock);
462 entry = get_unlocked_mapping_entry(mapping, index, NULL);
463 if (!entry || !radix_tree_exceptional_entry(entry))
464 goto out;
465 if (!trunc &&
466 (radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) ||
467 radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE)))
468 goto out;
469 radix_tree_delete(page_tree, index);
470 mapping->nrexceptional--;
471 ret = 1;
472out:
473 put_unlocked_mapping_entry(mapping, index, entry);
474 spin_unlock_irq(&mapping->tree_lock);
475 return ret;
476}
Jan Karaac401cc2016-05-12 18:29:18 +0200477/*
478 * Delete exceptional DAX entry at @index from @mapping. Wait for radix tree
479 * entry to get unlocked before deleting it.
480 */
481int dax_delete_mapping_entry(struct address_space *mapping, pgoff_t index)
482{
Jan Karac6dcf522016-08-10 17:22:44 +0200483 int ret = __dax_invalidate_mapping_entry(mapping, index, true);
Jan Karaac401cc2016-05-12 18:29:18 +0200484
Jan Karaac401cc2016-05-12 18:29:18 +0200485 /*
486 * This gets called from truncate / punch_hole path. As such, the caller
487 * must hold locks protecting against concurrent modifications of the
488 * radix tree (usually fs-private i_mmap_sem for writing). Since the
489 * caller has seen exceptional entry for this index, we better find it
490 * at that index as well...
491 */
Jan Karac6dcf522016-08-10 17:22:44 +0200492 WARN_ON_ONCE(!ret);
493 return ret;
494}
Jan Karaac401cc2016-05-12 18:29:18 +0200495
Jan Karac6dcf522016-08-10 17:22:44 +0200496/*
497 * Invalidate exceptional DAX entry if easily possible. This handles DAX
498 * entries for invalidate_inode_pages() so we evict the entry only if we can
499 * do so without blocking.
500 */
501int dax_invalidate_mapping_entry(struct address_space *mapping, pgoff_t index)
502{
503 int ret = 0;
504 void *entry, **slot;
505 struct radix_tree_root *page_tree = &mapping->page_tree;
506
507 spin_lock_irq(&mapping->tree_lock);
508 entry = __radix_tree_lookup(page_tree, index, NULL, &slot);
509 if (!entry || !radix_tree_exceptional_entry(entry) ||
510 slot_locked(mapping, slot))
511 goto out;
512 if (radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_DIRTY) ||
513 radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))
514 goto out;
515 radix_tree_delete(page_tree, index);
516 mapping->nrexceptional--;
517 ret = 1;
518out:
519 spin_unlock_irq(&mapping->tree_lock);
520 if (ret)
521 dax_wake_mapping_entry_waiter(mapping, index, entry, true);
522 return ret;
523}
524
525/*
526 * Invalidate exceptional DAX entry if it is clean.
527 */
528int dax_invalidate_mapping_entry_sync(struct address_space *mapping,
529 pgoff_t index)
530{
531 return __dax_invalidate_mapping_entry(mapping, index, false);
Jan Karaac401cc2016-05-12 18:29:18 +0200532}
533
534/*
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800535 * The user has performed a load from a hole in the file. Allocating
536 * a new page in the file would cause excessive storage usage for
537 * workloads with sparse files. We allocate a page cache page instead.
538 * We'll kick it out of the page cache if it's ever written to,
539 * otherwise it will simply fall out of the page cache under memory
540 * pressure without ever having been dirtied.
541 */
Jan Karaf449b932016-10-19 14:48:38 +0200542static int dax_load_hole(struct address_space *mapping, void **entry,
Jan Karaac401cc2016-05-12 18:29:18 +0200543 struct vm_fault *vmf)
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800544{
Jan Karaac401cc2016-05-12 18:29:18 +0200545 struct page *page;
Jan Karaf449b932016-10-19 14:48:38 +0200546 int ret;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800547
Jan Karaac401cc2016-05-12 18:29:18 +0200548 /* Hole page already exists? Return it... */
Jan Karaf449b932016-10-19 14:48:38 +0200549 if (!radix_tree_exceptional_entry(*entry)) {
550 page = *entry;
551 goto out;
Jan Karaac401cc2016-05-12 18:29:18 +0200552 }
553
554 /* This will replace locked radix tree entry with a hole page */
555 page = find_or_create_page(mapping, vmf->pgoff,
556 vmf->gfp_mask | __GFP_ZERO);
Jan Karab1aa8122016-12-14 15:07:24 -0800557 if (!page)
Jan Karaac401cc2016-05-12 18:29:18 +0200558 return VM_FAULT_OOM;
Jan Karaf449b932016-10-19 14:48:38 +0200559 out:
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800560 vmf->page = page;
Jan Karaf449b932016-10-19 14:48:38 +0200561 ret = finish_fault(vmf);
562 vmf->page = NULL;
563 *entry = page;
564 if (!ret) {
565 /* Grab reference for PTE that is now referencing the page */
566 get_page(page);
567 return VM_FAULT_NOPAGE;
568 }
569 return ret;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800570}
571
Christoph Hellwigb0d5e822016-09-19 11:24:49 +1000572static int copy_user_dax(struct block_device *bdev, sector_t sector, size_t size,
573 struct page *to, unsigned long vaddr)
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800574{
Dan Williamsb2e0d162016-01-15 16:55:59 -0800575 struct blk_dax_ctl dax = {
Christoph Hellwigb0d5e822016-09-19 11:24:49 +1000576 .sector = sector,
577 .size = size,
Dan Williamsb2e0d162016-01-15 16:55:59 -0800578 };
Ross Zwislere2e05392015-08-18 13:55:41 -0600579 void *vto;
580
Dan Williamsb2e0d162016-01-15 16:55:59 -0800581 if (dax_map_atomic(bdev, &dax) < 0)
582 return PTR_ERR(dax.addr);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800583 vto = kmap_atomic(to);
Dan Williamsb2e0d162016-01-15 16:55:59 -0800584 copy_user_page(vto, (void __force *)dax.addr, vaddr, to);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800585 kunmap_atomic(vto);
Dan Williamsb2e0d162016-01-15 16:55:59 -0800586 dax_unmap_atomic(bdev, &dax);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800587 return 0;
588}
589
Ross Zwisler642261a2016-11-08 11:34:45 +1100590/*
591 * By this point grab_mapping_entry() has ensured that we have a locked entry
592 * of the appropriate size so we don't have to worry about downgrading PMDs to
593 * PTEs. If we happen to be trying to insert a PTE and there is a PMD
594 * already in the tree, we will skip the insertion and just dirty the PMD as
595 * appropriate.
596 */
Jan Karaac401cc2016-05-12 18:29:18 +0200597static void *dax_insert_mapping_entry(struct address_space *mapping,
598 struct vm_fault *vmf,
Ross Zwisler642261a2016-11-08 11:34:45 +1100599 void *entry, sector_t sector,
600 unsigned long flags)
Ross Zwisler9973c982016-01-22 15:10:47 -0800601{
602 struct radix_tree_root *page_tree = &mapping->page_tree;
Jan Karaac401cc2016-05-12 18:29:18 +0200603 int error = 0;
604 bool hole_fill = false;
605 void *new_entry;
606 pgoff_t index = vmf->pgoff;
Ross Zwisler9973c982016-01-22 15:10:47 -0800607
Jan Karaac401cc2016-05-12 18:29:18 +0200608 if (vmf->flags & FAULT_FLAG_WRITE)
Dmitry Monakhovd2b2a282016-02-05 15:36:55 -0800609 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
Ross Zwisler9973c982016-01-22 15:10:47 -0800610
Jan Karaac401cc2016-05-12 18:29:18 +0200611 /* Replacing hole page with block mapping? */
612 if (!radix_tree_exceptional_entry(entry)) {
613 hole_fill = true;
614 /*
615 * Unmap the page now before we remove it from page cache below.
616 * The page is locked so it cannot be faulted in again.
617 */
618 unmap_mapping_range(mapping, vmf->pgoff << PAGE_SHIFT,
619 PAGE_SIZE, 0);
620 error = radix_tree_preload(vmf->gfp_mask & ~__GFP_HIGHMEM);
621 if (error)
622 return ERR_PTR(error);
Ross Zwisler642261a2016-11-08 11:34:45 +1100623 } else if (dax_is_zero_entry(entry) && !(flags & RADIX_DAX_HZP)) {
624 /* replacing huge zero page with PMD block mapping */
625 unmap_mapping_range(mapping,
626 (vmf->pgoff << PAGE_SHIFT) & PMD_MASK, PMD_SIZE, 0);
Ross Zwisler9973c982016-01-22 15:10:47 -0800627 }
628
Jan Karaac401cc2016-05-12 18:29:18 +0200629 spin_lock_irq(&mapping->tree_lock);
Ross Zwisler642261a2016-11-08 11:34:45 +1100630 new_entry = dax_radix_locked_entry(sector, flags);
631
Jan Karaac401cc2016-05-12 18:29:18 +0200632 if (hole_fill) {
633 __delete_from_page_cache(entry, NULL);
634 /* Drop pagecache reference */
635 put_page(entry);
Ross Zwisler642261a2016-11-08 11:34:45 +1100636 error = __radix_tree_insert(page_tree, index,
637 dax_radix_order(new_entry), new_entry);
Jan Karaac401cc2016-05-12 18:29:18 +0200638 if (error) {
639 new_entry = ERR_PTR(error);
Ross Zwisler9973c982016-01-22 15:10:47 -0800640 goto unlock;
641 }
Jan Karaac401cc2016-05-12 18:29:18 +0200642 mapping->nrexceptional++;
Ross Zwisler642261a2016-11-08 11:34:45 +1100643 } else if (dax_is_zero_entry(entry) || dax_is_empty_entry(entry)) {
644 /*
645 * Only swap our new entry into the radix tree if the current
646 * entry is a zero page or an empty entry. If a normal PTE or
647 * PMD entry is already in the tree, we leave it alone. This
648 * means that if we are trying to insert a PTE and the
649 * existing entry is a PMD, we will just leave the PMD in the
650 * tree and dirty it if necessary.
651 */
Johannes Weinerf7942432016-12-12 16:43:41 -0800652 struct radix_tree_node *node;
Jan Karaac401cc2016-05-12 18:29:18 +0200653 void **slot;
654 void *ret;
Ross Zwisler9973c982016-01-22 15:10:47 -0800655
Johannes Weinerf7942432016-12-12 16:43:41 -0800656 ret = __radix_tree_lookup(page_tree, index, &node, &slot);
Jan Karaac401cc2016-05-12 18:29:18 +0200657 WARN_ON_ONCE(ret != entry);
Johannes Weiner4d693d02016-12-12 16:43:49 -0800658 __radix_tree_replace(page_tree, node, slot,
659 new_entry, NULL, NULL);
Ross Zwisler9973c982016-01-22 15:10:47 -0800660 }
Jan Karaac401cc2016-05-12 18:29:18 +0200661 if (vmf->flags & FAULT_FLAG_WRITE)
Ross Zwisler9973c982016-01-22 15:10:47 -0800662 radix_tree_tag_set(page_tree, index, PAGECACHE_TAG_DIRTY);
663 unlock:
664 spin_unlock_irq(&mapping->tree_lock);
Jan Karaac401cc2016-05-12 18:29:18 +0200665 if (hole_fill) {
666 radix_tree_preload_end();
667 /*
668 * We don't need hole page anymore, it has been replaced with
669 * locked radix tree entry now.
670 */
671 if (mapping->a_ops->freepage)
672 mapping->a_ops->freepage(entry);
673 unlock_page(entry);
674 put_page(entry);
675 }
676 return new_entry;
Ross Zwisler9973c982016-01-22 15:10:47 -0800677}
678
Jan Kara4b4bb462016-12-14 15:07:53 -0800679static inline unsigned long
680pgoff_address(pgoff_t pgoff, struct vm_area_struct *vma)
681{
682 unsigned long address;
683
684 address = vma->vm_start + ((pgoff - vma->vm_pgoff) << PAGE_SHIFT);
685 VM_BUG_ON_VMA(address < vma->vm_start || address >= vma->vm_end, vma);
686 return address;
687}
688
689/* Walk all mappings of a given index of a file and writeprotect them */
690static void dax_mapping_entry_mkclean(struct address_space *mapping,
691 pgoff_t index, unsigned long pfn)
692{
693 struct vm_area_struct *vma;
694 pte_t *ptep;
695 pte_t pte;
696 spinlock_t *ptl;
697 bool changed;
698
699 i_mmap_lock_read(mapping);
700 vma_interval_tree_foreach(vma, &mapping->i_mmap, index, index) {
701 unsigned long address;
702
703 cond_resched();
704
705 if (!(vma->vm_flags & VM_SHARED))
706 continue;
707
708 address = pgoff_address(index, vma);
709 changed = false;
710 if (follow_pte(vma->vm_mm, address, &ptep, &ptl))
711 continue;
712 if (pfn != pte_pfn(*ptep))
713 goto unlock;
714 if (!pte_dirty(*ptep) && !pte_write(*ptep))
715 goto unlock;
716
717 flush_cache_page(vma, address, pfn);
718 pte = ptep_clear_flush(vma, address, ptep);
719 pte = pte_wrprotect(pte);
720 pte = pte_mkclean(pte);
721 set_pte_at(vma->vm_mm, address, ptep, pte);
722 changed = true;
723unlock:
724 pte_unmap_unlock(ptep, ptl);
725
726 if (changed)
727 mmu_notifier_invalidate_page(vma->vm_mm, address);
728 }
729 i_mmap_unlock_read(mapping);
730}
731
Ross Zwisler9973c982016-01-22 15:10:47 -0800732static int dax_writeback_one(struct block_device *bdev,
733 struct address_space *mapping, pgoff_t index, void *entry)
734{
735 struct radix_tree_root *page_tree = &mapping->page_tree;
Ross Zwisler9973c982016-01-22 15:10:47 -0800736 struct blk_dax_ctl dax;
Jan Karaa6abc2c2016-12-14 15:07:47 -0800737 void *entry2, **slot;
Ross Zwisler9973c982016-01-22 15:10:47 -0800738 int ret = 0;
739
Ross Zwisler9973c982016-01-22 15:10:47 -0800740 /*
Jan Karaa6abc2c2016-12-14 15:07:47 -0800741 * A page got tagged dirty in DAX mapping? Something is seriously
742 * wrong.
Ross Zwisler9973c982016-01-22 15:10:47 -0800743 */
Jan Karaa6abc2c2016-12-14 15:07:47 -0800744 if (WARN_ON(!radix_tree_exceptional_entry(entry)))
745 return -EIO;
Ross Zwisler9973c982016-01-22 15:10:47 -0800746
Jan Karaa6abc2c2016-12-14 15:07:47 -0800747 spin_lock_irq(&mapping->tree_lock);
748 entry2 = get_unlocked_mapping_entry(mapping, index, &slot);
749 /* Entry got punched out / reallocated? */
750 if (!entry2 || !radix_tree_exceptional_entry(entry2))
751 goto put_unlocked;
752 /*
753 * Entry got reallocated elsewhere? No need to writeback. We have to
754 * compare sectors as we must not bail out due to difference in lockbit
755 * or entry type.
756 */
757 if (dax_radix_sector(entry2) != dax_radix_sector(entry))
758 goto put_unlocked;
Ross Zwisler642261a2016-11-08 11:34:45 +1100759 if (WARN_ON_ONCE(dax_is_empty_entry(entry) ||
760 dax_is_zero_entry(entry))) {
Ross Zwisler9973c982016-01-22 15:10:47 -0800761 ret = -EIO;
Jan Karaa6abc2c2016-12-14 15:07:47 -0800762 goto put_unlocked;
Ross Zwisler9973c982016-01-22 15:10:47 -0800763 }
764
Jan Karaa6abc2c2016-12-14 15:07:47 -0800765 /* Another fsync thread may have already written back this entry */
766 if (!radix_tree_tag_get(page_tree, index, PAGECACHE_TAG_TOWRITE))
767 goto put_unlocked;
768 /* Lock the entry to serialize with page faults */
769 entry = lock_slot(mapping, slot);
770 /*
771 * We can clear the tag now but we have to be careful so that concurrent
772 * dax_writeback_one() calls for the same index cannot finish before we
773 * actually flush the caches. This is achieved as the calls will look
774 * at the entry only under tree_lock and once they do that they will
775 * see the entry locked and wait for it to unlock.
776 */
777 radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_TOWRITE);
778 spin_unlock_irq(&mapping->tree_lock);
779
Ross Zwisler642261a2016-11-08 11:34:45 +1100780 /*
781 * Even if dax_writeback_mapping_range() was given a wbc->range_start
782 * in the middle of a PMD, the 'index' we are given will be aligned to
783 * the start index of the PMD, as will the sector we pull from
784 * 'entry'. This allows us to flush for PMD_SIZE and not have to
785 * worry about partial PMD writebacks.
786 */
787 dax.sector = dax_radix_sector(entry);
788 dax.size = PAGE_SIZE << dax_radix_order(entry);
Ross Zwisler9973c982016-01-22 15:10:47 -0800789
790 /*
791 * We cannot hold tree_lock while calling dax_map_atomic() because it
792 * eventually calls cond_resched().
793 */
794 ret = dax_map_atomic(bdev, &dax);
Jan Karaa6abc2c2016-12-14 15:07:47 -0800795 if (ret < 0) {
796 put_locked_mapping_entry(mapping, index, entry);
Ross Zwisler9973c982016-01-22 15:10:47 -0800797 return ret;
Jan Karaa6abc2c2016-12-14 15:07:47 -0800798 }
Ross Zwisler9973c982016-01-22 15:10:47 -0800799
800 if (WARN_ON_ONCE(ret < dax.size)) {
801 ret = -EIO;
802 goto unmap;
803 }
804
Jan Kara4b4bb462016-12-14 15:07:53 -0800805 dax_mapping_entry_mkclean(mapping, index, pfn_t_to_pfn(dax.pfn));
Ross Zwisler9973c982016-01-22 15:10:47 -0800806 wb_cache_pmem(dax.addr, dax.size);
Jan Kara4b4bb462016-12-14 15:07:53 -0800807 /*
808 * After we have flushed the cache, we can clear the dirty tag. There
809 * cannot be new dirty data in the pfn after the flush has completed as
810 * the pfn mappings are writeprotected and fault waits for mapping
811 * entry lock.
812 */
813 spin_lock_irq(&mapping->tree_lock);
814 radix_tree_tag_clear(page_tree, index, PAGECACHE_TAG_DIRTY);
815 spin_unlock_irq(&mapping->tree_lock);
Ross Zwisler9973c982016-01-22 15:10:47 -0800816 unmap:
817 dax_unmap_atomic(bdev, &dax);
Jan Karaa6abc2c2016-12-14 15:07:47 -0800818 put_locked_mapping_entry(mapping, index, entry);
Ross Zwisler9973c982016-01-22 15:10:47 -0800819 return ret;
820
Jan Karaa6abc2c2016-12-14 15:07:47 -0800821 put_unlocked:
822 put_unlocked_mapping_entry(mapping, index, entry2);
Ross Zwisler9973c982016-01-22 15:10:47 -0800823 spin_unlock_irq(&mapping->tree_lock);
824 return ret;
825}
826
827/*
828 * Flush the mapping to the persistent domain within the byte range of [start,
829 * end]. This is required by data integrity operations to ensure file data is
830 * on persistent storage prior to completion of the operation.
831 */
Ross Zwisler7f6d5b52016-02-26 15:19:55 -0800832int dax_writeback_mapping_range(struct address_space *mapping,
833 struct block_device *bdev, struct writeback_control *wbc)
Ross Zwisler9973c982016-01-22 15:10:47 -0800834{
835 struct inode *inode = mapping->host;
Ross Zwisler642261a2016-11-08 11:34:45 +1100836 pgoff_t start_index, end_index;
Ross Zwisler9973c982016-01-22 15:10:47 -0800837 pgoff_t indices[PAGEVEC_SIZE];
838 struct pagevec pvec;
839 bool done = false;
840 int i, ret = 0;
Ross Zwisler9973c982016-01-22 15:10:47 -0800841
842 if (WARN_ON_ONCE(inode->i_blkbits != PAGE_SHIFT))
843 return -EIO;
844
Ross Zwisler7f6d5b52016-02-26 15:19:55 -0800845 if (!mapping->nrexceptional || wbc->sync_mode != WB_SYNC_ALL)
846 return 0;
847
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300848 start_index = wbc->range_start >> PAGE_SHIFT;
849 end_index = wbc->range_end >> PAGE_SHIFT;
Ross Zwisler9973c982016-01-22 15:10:47 -0800850
851 tag_pages_for_writeback(mapping, start_index, end_index);
852
853 pagevec_init(&pvec, 0);
854 while (!done) {
855 pvec.nr = find_get_entries_tag(mapping, start_index,
856 PAGECACHE_TAG_TOWRITE, PAGEVEC_SIZE,
857 pvec.pages, indices);
858
859 if (pvec.nr == 0)
860 break;
861
862 for (i = 0; i < pvec.nr; i++) {
863 if (indices[i] > end_index) {
864 done = true;
865 break;
866 }
867
868 ret = dax_writeback_one(bdev, mapping, indices[i],
869 pvec.pages[i]);
870 if (ret < 0)
871 return ret;
872 }
873 }
Ross Zwisler9973c982016-01-22 15:10:47 -0800874 return 0;
875}
876EXPORT_SYMBOL_GPL(dax_writeback_mapping_range);
877
Jan Karaac401cc2016-05-12 18:29:18 +0200878static int dax_insert_mapping(struct address_space *mapping,
Christoph Hellwig1aaba092016-09-19 11:24:49 +1000879 struct block_device *bdev, sector_t sector, size_t size,
880 void **entryp, struct vm_area_struct *vma, struct vm_fault *vmf)
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800881{
Jan Kara1a29d852016-12-14 15:07:01 -0800882 unsigned long vaddr = vmf->address;
Dan Williamsb2e0d162016-01-15 16:55:59 -0800883 struct blk_dax_ctl dax = {
Christoph Hellwig1aaba092016-09-19 11:24:49 +1000884 .sector = sector,
885 .size = size,
Dan Williamsb2e0d162016-01-15 16:55:59 -0800886 };
Jan Karaac401cc2016-05-12 18:29:18 +0200887 void *ret;
888 void *entry = *entryp;
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800889
Jan Kara4d9a2c82016-05-12 18:29:20 +0200890 if (dax_map_atomic(bdev, &dax) < 0)
891 return PTR_ERR(dax.addr);
Dan Williamsb2e0d162016-01-15 16:55:59 -0800892 dax_unmap_atomic(bdev, &dax);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800893
Ross Zwisler642261a2016-11-08 11:34:45 +1100894 ret = dax_insert_mapping_entry(mapping, vmf, entry, dax.sector, 0);
Jan Kara4d9a2c82016-05-12 18:29:20 +0200895 if (IS_ERR(ret))
896 return PTR_ERR(ret);
Jan Karaac401cc2016-05-12 18:29:18 +0200897 *entryp = ret;
Ross Zwisler9973c982016-01-22 15:10:47 -0800898
Jan Kara4d9a2c82016-05-12 18:29:20 +0200899 return vm_insert_mixed(vma, vaddr, dax.pfn);
Matthew Wilcoxf7ca90b2015-02-16 15:59:02 -0800900}
901
Dave Chinnerce5c5d52015-06-04 09:18:18 +1000902/**
Boaz Harrosh0e3b2102015-04-15 16:15:14 -0700903 * dax_pfn_mkwrite - handle first write to DAX page
904 * @vma: The virtual memory area where the fault occurred
905 * @vmf: The description of the fault
Boaz Harrosh0e3b2102015-04-15 16:15:14 -0700906 */
907int dax_pfn_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf)
908{
Ross Zwisler9973c982016-01-22 15:10:47 -0800909 struct file *file = vma->vm_file;
Jan Karaac401cc2016-05-12 18:29:18 +0200910 struct address_space *mapping = file->f_mapping;
Jan Kara2f89dc12016-12-14 15:07:50 -0800911 void *entry, **slot;
Jan Karaac401cc2016-05-12 18:29:18 +0200912 pgoff_t index = vmf->pgoff;
Boaz Harrosh0e3b2102015-04-15 16:15:14 -0700913
Jan Karaac401cc2016-05-12 18:29:18 +0200914 spin_lock_irq(&mapping->tree_lock);
Jan Kara2f89dc12016-12-14 15:07:50 -0800915 entry = get_unlocked_mapping_entry(mapping, index, &slot);
916 if (!entry || !radix_tree_exceptional_entry(entry)) {
917 if (entry)
918 put_unlocked_mapping_entry(mapping, index, entry);
919 spin_unlock_irq(&mapping->tree_lock);
920 return VM_FAULT_NOPAGE;
921 }
Jan Karaac401cc2016-05-12 18:29:18 +0200922 radix_tree_tag_set(&mapping->page_tree, index, PAGECACHE_TAG_DIRTY);
Jan Kara2f89dc12016-12-14 15:07:50 -0800923 entry = lock_slot(mapping, slot);
Jan Karaac401cc2016-05-12 18:29:18 +0200924 spin_unlock_irq(&mapping->tree_lock);
Jan Kara2f89dc12016-12-14 15:07:50 -0800925 /*
926 * If we race with somebody updating the PTE and finish_mkwrite_fault()
927 * fails, we don't care. We need to return VM_FAULT_NOPAGE and retry
928 * the fault in either case.
929 */
930 finish_mkwrite_fault(vmf);
931 put_locked_mapping_entry(mapping, index, entry);
Boaz Harrosh0e3b2102015-04-15 16:15:14 -0700932 return VM_FAULT_NOPAGE;
933}
934EXPORT_SYMBOL_GPL(dax_pfn_mkwrite);
935
Vishal Verma4b0228f2016-04-21 15:13:46 -0400936static bool dax_range_is_aligned(struct block_device *bdev,
937 unsigned int offset, unsigned int length)
938{
939 unsigned short sector_size = bdev_logical_block_size(bdev);
940
941 if (!IS_ALIGNED(offset, sector_size))
942 return false;
943 if (!IS_ALIGNED(length, sector_size))
944 return false;
945
946 return true;
947}
948
Christoph Hellwig679c8bd2016-05-09 10:47:04 +0200949int __dax_zero_page_range(struct block_device *bdev, sector_t sector,
950 unsigned int offset, unsigned int length)
951{
952 struct blk_dax_ctl dax = {
953 .sector = sector,
954 .size = PAGE_SIZE,
955 };
956
Vishal Verma4b0228f2016-04-21 15:13:46 -0400957 if (dax_range_is_aligned(bdev, offset, length)) {
958 sector_t start_sector = dax.sector + (offset >> 9);
959
960 return blkdev_issue_zeroout(bdev, start_sector,
961 length >> 9, GFP_NOFS, true);
962 } else {
963 if (dax_map_atomic(bdev, &dax) < 0)
964 return PTR_ERR(dax.addr);
965 clear_pmem(dax.addr + offset, length);
Vishal Verma4b0228f2016-04-21 15:13:46 -0400966 dax_unmap_atomic(bdev, &dax);
967 }
Christoph Hellwig679c8bd2016-05-09 10:47:04 +0200968 return 0;
969}
970EXPORT_SYMBOL_GPL(__dax_zero_page_range);
971
Christoph Hellwiga254e562016-09-19 11:24:49 +1000972#ifdef CONFIG_FS_IOMAP
Ross Zwisler333ccc92016-11-08 11:33:09 +1100973static sector_t dax_iomap_sector(struct iomap *iomap, loff_t pos)
974{
975 return iomap->blkno + (((pos & PAGE_MASK) - iomap->offset) >> 9);
976}
977
Christoph Hellwiga254e562016-09-19 11:24:49 +1000978static loff_t
Ross Zwisler11c59c92016-11-08 11:32:46 +1100979dax_iomap_actor(struct inode *inode, loff_t pos, loff_t length, void *data,
Christoph Hellwiga254e562016-09-19 11:24:49 +1000980 struct iomap *iomap)
981{
982 struct iov_iter *iter = data;
983 loff_t end = pos + length, done = 0;
984 ssize_t ret = 0;
985
986 if (iov_iter_rw(iter) == READ) {
987 end = min(end, i_size_read(inode));
988 if (pos >= end)
989 return 0;
990
991 if (iomap->type == IOMAP_HOLE || iomap->type == IOMAP_UNWRITTEN)
992 return iov_iter_zero(min(length, end - pos), iter);
993 }
994
995 if (WARN_ON_ONCE(iomap->type != IOMAP_MAPPED))
996 return -EIO;
997
Jan Karae3fce682016-08-10 17:10:28 +0200998 /*
999 * Write can allocate block for an area which has a hole page mapped
1000 * into page tables. We have to tear down these mappings so that data
1001 * written by write(2) is visible in mmap.
1002 */
1003 if ((iomap->flags & IOMAP_F_NEW) && inode->i_mapping->nrpages) {
1004 invalidate_inode_pages2_range(inode->i_mapping,
1005 pos >> PAGE_SHIFT,
1006 (end - 1) >> PAGE_SHIFT);
1007 }
1008
Christoph Hellwiga254e562016-09-19 11:24:49 +10001009 while (pos < end) {
1010 unsigned offset = pos & (PAGE_SIZE - 1);
1011 struct blk_dax_ctl dax = { 0 };
1012 ssize_t map_len;
1013
Ross Zwisler333ccc92016-11-08 11:33:09 +11001014 dax.sector = dax_iomap_sector(iomap, pos);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001015 dax.size = (length + offset + PAGE_SIZE - 1) & PAGE_MASK;
1016 map_len = dax_map_atomic(iomap->bdev, &dax);
1017 if (map_len < 0) {
1018 ret = map_len;
1019 break;
1020 }
1021
1022 dax.addr += offset;
1023 map_len -= offset;
1024 if (map_len > end - pos)
1025 map_len = end - pos;
1026
1027 if (iov_iter_rw(iter) == WRITE)
1028 map_len = copy_from_iter_pmem(dax.addr, map_len, iter);
1029 else
1030 map_len = copy_to_iter(dax.addr, map_len, iter);
1031 dax_unmap_atomic(iomap->bdev, &dax);
1032 if (map_len <= 0) {
1033 ret = map_len ? map_len : -EFAULT;
1034 break;
1035 }
1036
1037 pos += map_len;
1038 length -= map_len;
1039 done += map_len;
1040 }
1041
1042 return done ? done : ret;
1043}
1044
1045/**
Ross Zwisler11c59c92016-11-08 11:32:46 +11001046 * dax_iomap_rw - Perform I/O to a DAX file
Christoph Hellwiga254e562016-09-19 11:24:49 +10001047 * @iocb: The control block for this I/O
1048 * @iter: The addresses to do I/O from or to
1049 * @ops: iomap ops passed from the file system
1050 *
1051 * This function performs read and write operations to directly mapped
1052 * persistent memory. The callers needs to take care of read/write exclusion
1053 * and evicting any page cache pages in the region under I/O.
1054 */
1055ssize_t
Ross Zwisler11c59c92016-11-08 11:32:46 +11001056dax_iomap_rw(struct kiocb *iocb, struct iov_iter *iter,
Christoph Hellwiga254e562016-09-19 11:24:49 +10001057 struct iomap_ops *ops)
1058{
1059 struct address_space *mapping = iocb->ki_filp->f_mapping;
1060 struct inode *inode = mapping->host;
1061 loff_t pos = iocb->ki_pos, ret = 0, done = 0;
1062 unsigned flags = 0;
1063
1064 if (iov_iter_rw(iter) == WRITE)
1065 flags |= IOMAP_WRITE;
1066
Christoph Hellwiga254e562016-09-19 11:24:49 +10001067 while (iov_iter_count(iter)) {
1068 ret = iomap_apply(inode, pos, iov_iter_count(iter), flags, ops,
Ross Zwisler11c59c92016-11-08 11:32:46 +11001069 iter, dax_iomap_actor);
Christoph Hellwiga254e562016-09-19 11:24:49 +10001070 if (ret <= 0)
1071 break;
1072 pos += ret;
1073 done += ret;
1074 }
1075
1076 iocb->ki_pos += done;
1077 return done ? done : ret;
1078}
Ross Zwisler11c59c92016-11-08 11:32:46 +11001079EXPORT_SYMBOL_GPL(dax_iomap_rw);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001080
Jan Kara9f141d62016-10-19 14:34:31 +02001081static int dax_fault_return(int error)
1082{
1083 if (error == 0)
1084 return VM_FAULT_NOPAGE;
1085 if (error == -ENOMEM)
1086 return VM_FAULT_OOM;
1087 return VM_FAULT_SIGBUS;
1088}
1089
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001090/**
Ross Zwisler11c59c92016-11-08 11:32:46 +11001091 * dax_iomap_fault - handle a page fault on a DAX file
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001092 * @vma: The virtual memory area where the fault occurred
1093 * @vmf: The description of the fault
1094 * @ops: iomap ops passed from the file system
1095 *
1096 * When a page fault occurs, filesystems may call this helper in their fault
1097 * or mkwrite handler for DAX files. Assumes the caller has done all the
1098 * necessary locking for the page fault to proceed successfully.
1099 */
Ross Zwisler11c59c92016-11-08 11:32:46 +11001100int dax_iomap_fault(struct vm_area_struct *vma, struct vm_fault *vmf,
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001101 struct iomap_ops *ops)
1102{
1103 struct address_space *mapping = vma->vm_file->f_mapping;
1104 struct inode *inode = mapping->host;
Jan Kara1a29d852016-12-14 15:07:01 -08001105 unsigned long vaddr = vmf->address;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001106 loff_t pos = (loff_t)vmf->pgoff << PAGE_SHIFT;
1107 sector_t sector;
1108 struct iomap iomap = { 0 };
Jan Kara9484ab12016-11-10 10:26:50 +11001109 unsigned flags = IOMAP_FAULT;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001110 int error, major = 0;
Jan Karab1aa8122016-12-14 15:07:24 -08001111 int vmf_ret = 0;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001112 void *entry;
1113
1114 /*
1115 * Check whether offset isn't beyond end of file now. Caller is supposed
1116 * to hold locks serializing us with truncate / punch hole so this is
1117 * a reliable test.
1118 */
1119 if (pos >= i_size_read(inode))
1120 return VM_FAULT_SIGBUS;
1121
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001122 if ((vmf->flags & FAULT_FLAG_WRITE) && !vmf->cow_page)
1123 flags |= IOMAP_WRITE;
1124
1125 /*
1126 * Note that we don't bother to use iomap_apply here: DAX required
1127 * the file system block size to be equal the page size, which means
1128 * that we never have to deal with more than a single extent here.
1129 */
1130 error = ops->iomap_begin(inode, pos, PAGE_SIZE, flags, &iomap);
1131 if (error)
Jan Kara9f141d62016-10-19 14:34:31 +02001132 return dax_fault_return(error);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001133 if (WARN_ON_ONCE(iomap.offset + iomap.length < pos + PAGE_SIZE)) {
Jan Kara9f141d62016-10-19 14:34:31 +02001134 vmf_ret = dax_fault_return(-EIO); /* fs corruption? */
1135 goto finish_iomap;
1136 }
1137
1138 entry = grab_mapping_entry(mapping, vmf->pgoff, 0);
1139 if (IS_ERR(entry)) {
1140 vmf_ret = dax_fault_return(PTR_ERR(entry));
Ross Zwisler15502902016-11-08 11:33:26 +11001141 goto finish_iomap;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001142 }
1143
Ross Zwisler333ccc92016-11-08 11:33:09 +11001144 sector = dax_iomap_sector(&iomap, pos);
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001145
1146 if (vmf->cow_page) {
1147 switch (iomap.type) {
1148 case IOMAP_HOLE:
1149 case IOMAP_UNWRITTEN:
1150 clear_user_highpage(vmf->cow_page, vaddr);
1151 break;
1152 case IOMAP_MAPPED:
1153 error = copy_user_dax(iomap.bdev, sector, PAGE_SIZE,
1154 vmf->cow_page, vaddr);
1155 break;
1156 default:
1157 WARN_ON_ONCE(1);
1158 error = -EIO;
1159 break;
1160 }
1161
1162 if (error)
Jan Kara9f141d62016-10-19 14:34:31 +02001163 goto error_unlock_entry;
Jan Karab1aa8122016-12-14 15:07:24 -08001164
1165 __SetPageUptodate(vmf->cow_page);
1166 vmf_ret = finish_fault(vmf);
1167 if (!vmf_ret)
1168 vmf_ret = VM_FAULT_DONE_COW;
Jan Kara9f141d62016-10-19 14:34:31 +02001169 goto unlock_entry;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001170 }
1171
1172 switch (iomap.type) {
1173 case IOMAP_MAPPED:
1174 if (iomap.flags & IOMAP_F_NEW) {
1175 count_vm_event(PGMAJFAULT);
1176 mem_cgroup_count_vm_event(vma->vm_mm, PGMAJFAULT);
1177 major = VM_FAULT_MAJOR;
1178 }
1179 error = dax_insert_mapping(mapping, iomap.bdev, sector,
1180 PAGE_SIZE, &entry, vma, vmf);
Jan Kara9f141d62016-10-19 14:34:31 +02001181 /* -EBUSY is fine, somebody else faulted on the same PTE */
1182 if (error == -EBUSY)
1183 error = 0;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001184 break;
1185 case IOMAP_UNWRITTEN:
1186 case IOMAP_HOLE:
Ross Zwisler15502902016-11-08 11:33:26 +11001187 if (!(vmf->flags & FAULT_FLAG_WRITE)) {
Jan Karaf449b932016-10-19 14:48:38 +02001188 vmf_ret = dax_load_hole(mapping, &entry, vmf);
Jan Kara9f141d62016-10-19 14:34:31 +02001189 goto unlock_entry;
Ross Zwisler15502902016-11-08 11:33:26 +11001190 }
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001191 /*FALLTHRU*/
1192 default:
1193 WARN_ON_ONCE(1);
1194 error = -EIO;
1195 break;
1196 }
1197
Jan Kara9f141d62016-10-19 14:34:31 +02001198 error_unlock_entry:
1199 vmf_ret = dax_fault_return(error) | major;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001200 unlock_entry:
Jan Karaf449b932016-10-19 14:48:38 +02001201 put_locked_mapping_entry(mapping, vmf->pgoff, entry);
Jan Kara9f141d62016-10-19 14:34:31 +02001202 finish_iomap:
1203 if (ops->iomap_end) {
1204 int copied = PAGE_SIZE;
1205
1206 if (vmf_ret & VM_FAULT_ERROR)
1207 copied = 0;
1208 /*
1209 * The fault is done by now and there's no way back (other
1210 * thread may be already happily using PTE we have installed).
1211 * Just ignore error from ->iomap_end since we cannot do much
1212 * with it.
1213 */
1214 ops->iomap_end(inode, pos, PAGE_SIZE, copied, flags, &iomap);
Ross Zwisler15502902016-11-08 11:33:26 +11001215 }
Jan Kara9f141d62016-10-19 14:34:31 +02001216 return vmf_ret;
Christoph Hellwiga7d73fe2016-09-19 11:24:50 +10001217}
Ross Zwisler11c59c92016-11-08 11:32:46 +11001218EXPORT_SYMBOL_GPL(dax_iomap_fault);
Ross Zwisler642261a2016-11-08 11:34:45 +11001219
1220#ifdef CONFIG_FS_DAX_PMD
1221/*
1222 * The 'colour' (ie low bits) within a PMD of a page offset. This comes up
1223 * more often than one might expect in the below functions.
1224 */
1225#define PG_PMD_COLOUR ((PMD_SIZE >> PAGE_SHIFT) - 1)
1226
1227static int dax_pmd_insert_mapping(struct vm_area_struct *vma, pmd_t *pmd,
1228 struct vm_fault *vmf, unsigned long address,
1229 struct iomap *iomap, loff_t pos, bool write, void **entryp)
1230{
1231 struct address_space *mapping = vma->vm_file->f_mapping;
1232 struct block_device *bdev = iomap->bdev;
1233 struct blk_dax_ctl dax = {
1234 .sector = dax_iomap_sector(iomap, pos),
1235 .size = PMD_SIZE,
1236 };
1237 long length = dax_map_atomic(bdev, &dax);
1238 void *ret;
1239
1240 if (length < 0) /* dax_map_atomic() failed */
1241 return VM_FAULT_FALLBACK;
1242 if (length < PMD_SIZE)
1243 goto unmap_fallback;
1244 if (pfn_t_to_pfn(dax.pfn) & PG_PMD_COLOUR)
1245 goto unmap_fallback;
1246 if (!pfn_t_devmap(dax.pfn))
1247 goto unmap_fallback;
1248
1249 dax_unmap_atomic(bdev, &dax);
1250
1251 ret = dax_insert_mapping_entry(mapping, vmf, *entryp, dax.sector,
1252 RADIX_DAX_PMD);
1253 if (IS_ERR(ret))
1254 return VM_FAULT_FALLBACK;
1255 *entryp = ret;
1256
1257 return vmf_insert_pfn_pmd(vma, address, pmd, dax.pfn, write);
1258
1259 unmap_fallback:
1260 dax_unmap_atomic(bdev, &dax);
1261 return VM_FAULT_FALLBACK;
1262}
1263
1264static int dax_pmd_load_hole(struct vm_area_struct *vma, pmd_t *pmd,
1265 struct vm_fault *vmf, unsigned long address,
1266 struct iomap *iomap, void **entryp)
1267{
1268 struct address_space *mapping = vma->vm_file->f_mapping;
1269 unsigned long pmd_addr = address & PMD_MASK;
1270 struct page *zero_page;
1271 spinlock_t *ptl;
1272 pmd_t pmd_entry;
1273 void *ret;
1274
1275 zero_page = mm_get_huge_zero_page(vma->vm_mm);
1276
1277 if (unlikely(!zero_page))
1278 return VM_FAULT_FALLBACK;
1279
1280 ret = dax_insert_mapping_entry(mapping, vmf, *entryp, 0,
1281 RADIX_DAX_PMD | RADIX_DAX_HZP);
1282 if (IS_ERR(ret))
1283 return VM_FAULT_FALLBACK;
1284 *entryp = ret;
1285
1286 ptl = pmd_lock(vma->vm_mm, pmd);
1287 if (!pmd_none(*pmd)) {
1288 spin_unlock(ptl);
1289 return VM_FAULT_FALLBACK;
1290 }
1291
1292 pmd_entry = mk_pmd(zero_page, vma->vm_page_prot);
1293 pmd_entry = pmd_mkhuge(pmd_entry);
1294 set_pmd_at(vma->vm_mm, pmd_addr, pmd, pmd_entry);
1295 spin_unlock(ptl);
1296 return VM_FAULT_NOPAGE;
1297}
1298
1299int dax_iomap_pmd_fault(struct vm_area_struct *vma, unsigned long address,
1300 pmd_t *pmd, unsigned int flags, struct iomap_ops *ops)
1301{
1302 struct address_space *mapping = vma->vm_file->f_mapping;
1303 unsigned long pmd_addr = address & PMD_MASK;
1304 bool write = flags & FAULT_FLAG_WRITE;
Jan Kara9484ab12016-11-10 10:26:50 +11001305 unsigned int iomap_flags = (write ? IOMAP_WRITE : 0) | IOMAP_FAULT;
Ross Zwisler642261a2016-11-08 11:34:45 +11001306 struct inode *inode = mapping->host;
1307 int result = VM_FAULT_FALLBACK;
1308 struct iomap iomap = { 0 };
1309 pgoff_t max_pgoff, pgoff;
1310 struct vm_fault vmf;
1311 void *entry;
1312 loff_t pos;
1313 int error;
1314
1315 /* Fall back to PTEs if we're going to COW */
1316 if (write && !(vma->vm_flags & VM_SHARED))
1317 goto fallback;
1318
1319 /* If the PMD would extend outside the VMA */
1320 if (pmd_addr < vma->vm_start)
1321 goto fallback;
1322 if ((pmd_addr + PMD_SIZE) > vma->vm_end)
1323 goto fallback;
1324
1325 /*
1326 * Check whether offset isn't beyond end of file now. Caller is
1327 * supposed to hold locks serializing us with truncate / punch hole so
1328 * this is a reliable test.
1329 */
1330 pgoff = linear_page_index(vma, pmd_addr);
1331 max_pgoff = (i_size_read(inode) - 1) >> PAGE_SHIFT;
1332
1333 if (pgoff > max_pgoff)
1334 return VM_FAULT_SIGBUS;
1335
1336 /* If the PMD would extend beyond the file size */
1337 if ((pgoff | PG_PMD_COLOUR) > max_pgoff)
1338 goto fallback;
1339
1340 /*
Ross Zwisler642261a2016-11-08 11:34:45 +11001341 * Note that we don't use iomap_apply here. We aren't doing I/O, only
1342 * setting up a mapping, so really we're using iomap_begin() as a way
1343 * to look up our filesystem block.
1344 */
1345 pos = (loff_t)pgoff << PAGE_SHIFT;
1346 error = ops->iomap_begin(inode, pos, PMD_SIZE, iomap_flags, &iomap);
1347 if (error)
Jan Kara9f141d62016-10-19 14:34:31 +02001348 goto fallback;
1349
Ross Zwisler642261a2016-11-08 11:34:45 +11001350 if (iomap.offset + iomap.length < pos + PMD_SIZE)
1351 goto finish_iomap;
1352
Jan Kara9f141d62016-10-19 14:34:31 +02001353 /*
1354 * grab_mapping_entry() will make sure we get a 2M empty entry, a DAX
1355 * PMD or a HZP entry. If it can't (because a 4k page is already in
1356 * the tree, for instance), it will return -EEXIST and we just fall
1357 * back to 4k entries.
1358 */
1359 entry = grab_mapping_entry(mapping, pgoff, RADIX_DAX_PMD);
1360 if (IS_ERR(entry))
1361 goto finish_iomap;
1362
Ross Zwisler642261a2016-11-08 11:34:45 +11001363 vmf.pgoff = pgoff;
1364 vmf.flags = flags;
1365 vmf.gfp_mask = mapping_gfp_mask(mapping) | __GFP_IO;
1366
1367 switch (iomap.type) {
1368 case IOMAP_MAPPED:
1369 result = dax_pmd_insert_mapping(vma, pmd, &vmf, address,
1370 &iomap, pos, write, &entry);
1371 break;
1372 case IOMAP_UNWRITTEN:
1373 case IOMAP_HOLE:
1374 if (WARN_ON_ONCE(write))
Jan Kara9f141d62016-10-19 14:34:31 +02001375 goto unlock_entry;
Ross Zwisler642261a2016-11-08 11:34:45 +11001376 result = dax_pmd_load_hole(vma, pmd, &vmf, address, &iomap,
1377 &entry);
1378 break;
1379 default:
1380 WARN_ON_ONCE(1);
1381 break;
1382 }
1383
Ross Zwisler642261a2016-11-08 11:34:45 +11001384 unlock_entry:
1385 put_locked_mapping_entry(mapping, pgoff, entry);
Jan Kara9f141d62016-10-19 14:34:31 +02001386 finish_iomap:
1387 if (ops->iomap_end) {
1388 int copied = PMD_SIZE;
1389
1390 if (result == VM_FAULT_FALLBACK)
1391 copied = 0;
1392 /*
1393 * The fault is done by now and there's no way back (other
1394 * thread may be already happily using PMD we have installed).
1395 * Just ignore error from ->iomap_end since we cannot do much
1396 * with it.
1397 */
1398 ops->iomap_end(inode, pos, PMD_SIZE, copied, iomap_flags,
1399 &iomap);
1400 }
Ross Zwisler642261a2016-11-08 11:34:45 +11001401 fallback:
1402 if (result == VM_FAULT_FALLBACK) {
1403 split_huge_pmd(vma, pmd, address);
1404 count_vm_event(THP_FAULT_FALLBACK);
1405 }
1406 return result;
1407}
1408EXPORT_SYMBOL_GPL(dax_iomap_pmd_fault);
1409#endif /* CONFIG_FS_DAX_PMD */
Christoph Hellwiga254e562016-09-19 11:24:49 +10001410#endif /* CONFIG_FS_IOMAP */