blob: 5a8ae2125b50855d6f9f304ffd737fbbb57bb02a [file] [log] [blame]
Ricardo Labiaga85e174b2010-10-20 00:17:58 -04001/*
2 * pNFS functions to call and manage layout drivers.
3 *
4 * Copyright (c) 2002 [year of first publication]
5 * The Regents of the University of Michigan
6 * All Rights Reserved
7 *
8 * Dean Hildebrand <dhildebz@umich.edu>
9 *
10 * Permission is granted to use, copy, create derivative works, and
11 * redistribute this software and such derivative works for any purpose,
12 * so long as the name of the University of Michigan is not used in
13 * any advertising or publicity pertaining to the use or distribution
14 * of this software without specific, written prior authorization. If
15 * the above copyright notice or any other identification of the
16 * University of Michigan is included in any copy of any portion of
17 * this software, then the disclaimer below must also be included.
18 *
19 * This software is provided as is, without representation or warranty
20 * of any kind either express or implied, including without limitation
21 * the implied warranties of merchantability, fitness for a particular
22 * purpose, or noninfringement. The Regents of the University of
23 * Michigan shall not be liable for any damages, including special,
24 * indirect, incidental, or consequential damages, with respect to any
25 * claim arising out of or in connection with the use of the software,
26 * even if it has been or is hereafter advised of the possibility of
27 * such damages.
28 */
29
30#include <linux/nfs_fs.h>
Trond Myklebust493292d2011-07-13 15:58:28 -040031#include <linux/nfs_page.h>
Paul Gortmaker143cb492011-07-01 14:23:34 -040032#include <linux/module.h>
Andy Adamson974cec82010-10-20 00:18:02 -040033#include "internal.h"
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040034#include "pnfs.h"
Andy Adamson64419a92011-03-01 01:34:16 +000035#include "iostat.h"
Trond Myklebustcc668ab2013-08-14 15:31:28 -040036#include "nfs4trace.h"
Trond Myklebust40dd4b72015-01-24 13:54:37 -050037#include "delegation.h"
Peng Tao87334082015-06-23 19:51:57 +080038#include "nfs42.h"
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040039
40#define NFSDBG_FACILITY NFSDBG_PNFS
Trond Myklebust25c75332012-09-18 17:01:12 -040041#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040042
Fred Isaman02c35fc2010-10-20 00:17:59 -040043/* Locking:
44 *
45 * pnfs_spinlock:
46 * protects pnfs_modules_tbl.
47 */
48static DEFINE_SPINLOCK(pnfs_spinlock);
49
50/*
51 * pnfs_modules_tbl holds all pnfs modules
52 */
53static LIST_HEAD(pnfs_modules_tbl);
54
Peng Taoaa1e0e3a2014-09-06 00:53:25 +080055static int
56pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, nfs4_stateid stateid,
Peng Tao6c166052014-11-17 09:30:40 +080057 enum pnfs_iomode iomode, bool sync);
Peng Taoaa1e0e3a2014-09-06 00:53:25 +080058
Fred Isaman02c35fc2010-10-20 00:17:59 -040059/* Return the registered pnfs layout driver module matching given id */
60static struct pnfs_layoutdriver_type *
61find_pnfs_driver_locked(u32 id)
62{
63 struct pnfs_layoutdriver_type *local;
64
65 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid)
66 if (local->id == id)
67 goto out;
68 local = NULL;
69out:
70 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local);
71 return local;
72}
73
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040074static struct pnfs_layoutdriver_type *
75find_pnfs_driver(u32 id)
76{
Fred Isaman02c35fc2010-10-20 00:17:59 -040077 struct pnfs_layoutdriver_type *local;
78
79 spin_lock(&pnfs_spinlock);
80 local = find_pnfs_driver_locked(id);
Trond Myklebust0a9c63f2012-06-15 13:02:58 -040081 if (local != NULL && !try_module_get(local->owner)) {
82 dprintk("%s: Could not grab reference on module\n", __func__);
83 local = NULL;
84 }
Fred Isaman02c35fc2010-10-20 00:17:59 -040085 spin_unlock(&pnfs_spinlock);
86 return local;
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040087}
88
89void
90unset_pnfs_layoutdriver(struct nfs_server *nfss)
91{
Benny Halevy738fd0f32011-07-30 20:52:36 -040092 if (nfss->pnfs_curr_ld) {
93 if (nfss->pnfs_curr_ld->clear_layoutdriver)
94 nfss->pnfs_curr_ld->clear_layoutdriver(nfss);
Trond Myklebust2a4c8992012-06-14 13:08:38 -040095 /* Decrement the MDS count. Purge the deviceid cache if zero */
96 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count))
97 nfs4_deviceid_purge_client(nfss->nfs_client);
Fred Isaman02c35fc2010-10-20 00:17:59 -040098 module_put(nfss->pnfs_curr_ld->owner);
Benny Halevy738fd0f32011-07-30 20:52:36 -040099 }
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400100 nfss->pnfs_curr_ld = NULL;
101}
102
103/*
104 * Try to set the server's pnfs module to the pnfs layout type specified by id.
105 * Currently only one pNFS layout driver per filesystem is supported.
106 *
107 * @id layout type. Zero (illegal layout type) indicates pNFS not in use.
108 */
109void
Benny Halevy738fd0f32011-07-30 20:52:36 -0400110set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh,
111 u32 id)
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400112{
113 struct pnfs_layoutdriver_type *ld_type = NULL;
114
115 if (id == 0)
116 goto out_no_driver;
117 if (!(server->nfs_client->cl_exchange_flags &
118 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500119 printk(KERN_ERR "NFS: %s: id %u cl_exchange_flags 0x%x\n",
120 __func__, id, server->nfs_client->cl_exchange_flags);
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400121 goto out_no_driver;
122 }
123 ld_type = find_pnfs_driver(id);
124 if (!ld_type) {
125 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX, id);
126 ld_type = find_pnfs_driver(id);
127 if (!ld_type) {
128 dprintk("%s: No pNFS module found for %u.\n",
129 __func__, id);
130 goto out_no_driver;
131 }
132 }
133 server->pnfs_curr_ld = ld_type;
Benny Halevy738fd0f32011-07-30 20:52:36 -0400134 if (ld_type->set_layoutdriver
135 && ld_type->set_layoutdriver(server, mntfh)) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500136 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout "
137 "driver %u.\n", __func__, id);
Benny Halevy738fd0f32011-07-30 20:52:36 -0400138 module_put(ld_type->owner);
139 goto out_no_driver;
140 }
Trond Myklebust2a4c8992012-06-14 13:08:38 -0400141 /* Bump the MDS count */
142 atomic_inc(&server->nfs_client->cl_mds_count);
Christoph Hellwigea8eecd2011-03-01 01:34:21 +0000143
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400144 dprintk("%s: pNFS module for %u set\n", __func__, id);
145 return;
146
147out_no_driver:
148 dprintk("%s: Using NFSv4 I/O\n", __func__);
149 server->pnfs_curr_ld = NULL;
150}
Fred Isaman02c35fc2010-10-20 00:17:59 -0400151
152int
153pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
154{
155 int status = -EINVAL;
156 struct pnfs_layoutdriver_type *tmp;
157
158 if (ld_type->id == 0) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500159 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__);
Fred Isaman02c35fc2010-10-20 00:17:59 -0400160 return status;
161 }
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400162 if (!ld_type->alloc_lseg || !ld_type->free_lseg) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500163 printk(KERN_ERR "NFS: %s Layout driver must provide "
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400164 "alloc_lseg and free_lseg.\n", __func__);
165 return status;
166 }
Fred Isaman02c35fc2010-10-20 00:17:59 -0400167
168 spin_lock(&pnfs_spinlock);
169 tmp = find_pnfs_driver_locked(ld_type->id);
170 if (!tmp) {
171 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl);
172 status = 0;
173 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id,
174 ld_type->name);
175 } else {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500176 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n",
Fred Isaman02c35fc2010-10-20 00:17:59 -0400177 __func__, ld_type->id);
178 }
179 spin_unlock(&pnfs_spinlock);
180
181 return status;
182}
183EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver);
184
185void
186pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
187{
188 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id);
189 spin_lock(&pnfs_spinlock);
190 list_del(&ld_type->pnfs_tblid);
191 spin_unlock(&pnfs_spinlock);
192}
193EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);
Benny Halevye5e94012010-10-20 00:18:01 -0400194
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400195/*
196 * pNFS client layout cache
197 */
198
Fred Isamancc6e5342011-01-06 11:36:28 +0000199/* Need to hold i_lock if caller does not already hold reference */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000200void
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400201pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
Benny Halevye5e94012010-10-20 00:18:01 -0400202{
Fred Isamancc6e5342011-01-06 11:36:28 +0000203 atomic_inc(&lo->plh_refcount);
204}
205
Benny Halevy636fb9c2011-05-22 19:51:33 +0300206static struct pnfs_layout_hdr *
207pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
208{
209 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
Trond Myklebust57934272012-09-20 20:37:23 -0400210 return ld->alloc_layout_hdr(ino, gfp_flags);
Benny Halevy636fb9c2011-05-22 19:51:33 +0300211}
212
213static void
214pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
215{
Trond Myklebust9c626382012-09-20 17:31:43 -0400216 struct nfs_server *server = NFS_SERVER(lo->plh_inode);
217 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;
218
219 if (!list_empty(&lo->plh_layouts)) {
220 struct nfs_client *clp = server->nfs_client;
221
222 spin_lock(&clp->cl_lock);
223 list_del_init(&lo->plh_layouts);
224 spin_unlock(&clp->cl_lock);
225 }
Peng Tao9fa40752011-07-30 20:52:32 -0400226 put_rpccred(lo->plh_lc_cred);
Trond Myklebust57934272012-09-20 20:37:23 -0400227 return ld->free_layout_hdr(lo);
Benny Halevy636fb9c2011-05-22 19:51:33 +0300228}
229
Fred Isamancc6e5342011-01-06 11:36:28 +0000230static void
Trond Myklebust6622c3e2012-09-20 17:23:11 -0400231pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
Fred Isamancc6e5342011-01-06 11:36:28 +0000232{
Trond Myklebustbb346f62012-09-20 15:52:13 -0400233 struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
Fred Isamancc6e5342011-01-06 11:36:28 +0000234 dprintk("%s: freeing layout cache %p\n", __func__, lo);
Trond Myklebustbb346f62012-09-20 15:52:13 -0400235 nfsi->layout = NULL;
236 /* Reset MDS Threshold I/O counters */
237 nfsi->write_io = 0;
238 nfsi->read_io = 0;
Benny Halevye5e94012010-10-20 00:18:01 -0400239}
240
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400241void
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400242pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
Andy Adamson974cec82010-10-20 00:18:02 -0400243{
Fred Isamancc6e5342011-01-06 11:36:28 +0000244 struct inode *inode = lo->plh_inode;
245
246 if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
Peng Tao566f8732014-10-10 23:25:46 +0800247 if (!list_empty(&lo->plh_segs))
248 WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n");
Trond Myklebust6622c3e2012-09-20 17:23:11 -0400249 pnfs_detach_layout_hdr(lo);
Fred Isamancc6e5342011-01-06 11:36:28 +0000250 spin_unlock(&inode->i_lock);
Trond Myklebust6622c3e2012-09-20 17:23:11 -0400251 pnfs_free_layout_hdr(lo);
Fred Isamancc6e5342011-01-06 11:36:28 +0000252 }
Andy Adamson974cec82010-10-20 00:18:02 -0400253}
254
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400255static int
256pnfs_iomode_to_fail_bit(u32 iomode)
257{
258 return iomode == IOMODE_RW ?
259 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
260}
261
262static void
Trond Myklebust3e621212012-09-24 13:07:16 -0400263pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400264{
Trond Myklebust25c75332012-09-18 17:01:12 -0400265 lo->plh_retry_timestamp = jiffies;
Yanchuan Nian39e88fc2013-01-04 20:19:49 +0800266 if (!test_and_set_bit(fail_bit, &lo->plh_flags))
Trond Myklebust3e621212012-09-24 13:07:16 -0400267 atomic_inc(&lo->plh_refcount);
268}
269
270static void
271pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
272{
273 if (test_and_clear_bit(fail_bit, &lo->plh_flags))
274 atomic_dec(&lo->plh_refcount);
275}
276
277static void
278pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
279{
280 struct inode *inode = lo->plh_inode;
Trond Myklebust115ce572012-09-20 21:19:43 -0400281 struct pnfs_layout_range range = {
282 .iomode = iomode,
283 .offset = 0,
284 .length = NFS4_MAX_UINT64,
285 };
286 LIST_HEAD(head);
Trond Myklebust3e621212012-09-24 13:07:16 -0400287
288 spin_lock(&inode->i_lock);
289 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
Trond Myklebust115ce572012-09-20 21:19:43 -0400290 pnfs_mark_matching_lsegs_invalid(lo, &head, &range);
Trond Myklebust3e621212012-09-24 13:07:16 -0400291 spin_unlock(&inode->i_lock);
Trond Myklebust115ce572012-09-20 21:19:43 -0400292 pnfs_free_lseg_list(&head);
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400293 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
294 iomode == IOMODE_RW ? "RW" : "READ");
295}
296
297static bool
298pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
299{
Trond Myklebust25c75332012-09-18 17:01:12 -0400300 unsigned long start, end;
Trond Myklebust3e621212012-09-24 13:07:16 -0400301 int fail_bit = pnfs_iomode_to_fail_bit(iomode);
302
303 if (test_bit(fail_bit, &lo->plh_flags) == 0)
Trond Myklebust25c75332012-09-18 17:01:12 -0400304 return false;
305 end = jiffies;
306 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
307 if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
308 /* It is time to retry the failed layoutgets */
Trond Myklebust3e621212012-09-24 13:07:16 -0400309 pnfs_layout_clear_fail_bit(lo, fail_bit);
Trond Myklebust25c75332012-09-18 17:01:12 -0400310 return false;
311 }
312 return true;
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400313}
314
Andy Adamson974cec82010-10-20 00:18:02 -0400315static void
316init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg)
317{
Fred Isaman566052c2011-01-06 11:36:20 +0000318 INIT_LIST_HEAD(&lseg->pls_list);
Peng Taoa9bae562011-07-30 20:52:33 -0400319 INIT_LIST_HEAD(&lseg->pls_lc_list);
Fred Isaman4541d162011-01-06 11:36:23 +0000320 atomic_set(&lseg->pls_refcount, 1);
321 smp_mb();
322 set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
Fred Isaman566052c2011-01-06 11:36:20 +0000323 lseg->pls_layout = lo;
Andy Adamson974cec82010-10-20 00:18:02 -0400324}
325
Trond Myklebust905ca192012-09-20 20:46:49 -0400326static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
Andy Adamson974cec82010-10-20 00:18:02 -0400327{
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000328 struct inode *ino = lseg->pls_layout->plh_inode;
Andy Adamson974cec82010-10-20 00:18:02 -0400329
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400330 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
Andy Adamson974cec82010-10-20 00:18:02 -0400331}
332
Fred Isamand684d2a2011-03-01 01:34:13 +0000333static void
Trond Myklebust57036a32012-09-20 16:33:30 -0400334pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
335 struct pnfs_layout_segment *lseg)
Andy Adamson974cec82010-10-20 00:18:02 -0400336{
Trond Myklebust57036a32012-09-20 16:33:30 -0400337 struct inode *inode = lo->plh_inode;
Fred Isamand684d2a2011-03-01 01:34:13 +0000338
Benny Halevyd20581a2011-05-22 19:52:03 +0300339 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
Fred Isamand684d2a2011-03-01 01:34:13 +0000340 list_del_init(&lseg->pls_list);
Trond Myklebust8f0d27d2012-09-20 20:57:11 -0400341 /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
342 atomic_dec(&lo->plh_refcount);
Trond Myklebust173f77e2012-09-21 15:49:42 -0400343 if (list_empty(&lo->plh_segs))
344 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
Fred Isamand684d2a2011-03-01 01:34:13 +0000345 rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
346}
347
Peng Taoaa1e0e3a2014-09-06 00:53:25 +0800348/* Return true if layoutreturn is needed */
349static bool
350pnfs_layout_need_return(struct pnfs_layout_hdr *lo,
Peng Tao27b6f532014-10-20 14:44:38 +0800351 struct pnfs_layout_segment *lseg)
Peng Taoaa1e0e3a2014-09-06 00:53:25 +0800352{
353 struct pnfs_layout_segment *s;
354
Trond Myklebust7f273922015-07-09 18:40:01 +0200355 if (!test_and_clear_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
Peng Taoaa1e0e3a2014-09-06 00:53:25 +0800356 return false;
357
358 list_for_each_entry(s, &lo->plh_segs, pls_list)
Peng Tao27b6f532014-10-20 14:44:38 +0800359 if (s != lseg && test_bit(NFS_LSEG_LAYOUTRETURN, &s->pls_flags))
Peng Taoaa1e0e3a2014-09-06 00:53:25 +0800360 return false;
361
Peng Taoaa1e0e3a2014-09-06 00:53:25 +0800362 return true;
363}
364
Trond Myklebust7f273922015-07-09 18:40:01 +0200365static bool
366pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo)
367{
368 if (test_and_set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
369 return false;
370 lo->plh_return_iomode = 0;
Trond Myklebust7f273922015-07-09 18:40:01 +0200371 pnfs_get_layout_hdr(lo);
372 clear_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE, &lo->plh_flags);
373 return true;
374}
375
Trond Myklebust4ef2e4f2015-02-05 17:27:39 -0500376static void pnfs_layoutreturn_before_put_lseg(struct pnfs_layout_segment *lseg,
377 struct pnfs_layout_hdr *lo, struct inode *inode)
Peng Tao27b6f532014-10-20 14:44:38 +0800378{
Peng Tao27b6f532014-10-20 14:44:38 +0800379 lo = lseg->pls_layout;
380 inode = lo->plh_inode;
381
382 spin_lock(&inode->i_lock);
383 if (pnfs_layout_need_return(lo, lseg)) {
384 nfs4_stateid stateid;
385 enum pnfs_iomode iomode;
Trond Myklebust7f273922015-07-09 18:40:01 +0200386 bool send;
Peng Tao27b6f532014-10-20 14:44:38 +0800387
388 stateid = lo->plh_stateid;
389 iomode = lo->plh_return_iomode;
Trond Myklebust7f273922015-07-09 18:40:01 +0200390 send = pnfs_prepare_layoutreturn(lo);
Peng Tao27b6f532014-10-20 14:44:38 +0800391 spin_unlock(&inode->i_lock);
Trond Myklebust7f273922015-07-09 18:40:01 +0200392 if (send) {
393 /* Send an async layoutreturn so we dont deadlock */
394 pnfs_send_layoutreturn(lo, stateid, iomode, false);
395 }
Peng Tao27b6f532014-10-20 14:44:38 +0800396 } else
Trond Myklebust4ef2e4f2015-02-05 17:27:39 -0500397 spin_unlock(&inode->i_lock);
Peng Tao27b6f532014-10-20 14:44:38 +0800398}
399
Fred Isamanbae724e2011-03-01 01:34:15 +0000400void
Trond Myklebust9369a432012-09-18 20:57:08 -0400401pnfs_put_lseg(struct pnfs_layout_segment *lseg)
Fred Isamand684d2a2011-03-01 01:34:13 +0000402{
Trond Myklebust57036a32012-09-20 16:33:30 -0400403 struct pnfs_layout_hdr *lo;
Fred Isamand684d2a2011-03-01 01:34:13 +0000404 struct inode *inode;
405
406 if (!lseg)
407 return;
408
Fred Isaman4541d162011-01-06 11:36:23 +0000409 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
410 atomic_read(&lseg->pls_refcount),
411 test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
Trond Myklebust4ef2e4f2015-02-05 17:27:39 -0500412
413 /* Handle the case where refcount != 1 */
414 if (atomic_add_unless(&lseg->pls_refcount, -1, 1))
415 return;
416
Trond Myklebust57036a32012-09-20 16:33:30 -0400417 lo = lseg->pls_layout;
418 inode = lo->plh_inode;
Trond Myklebust4ef2e4f2015-02-05 17:27:39 -0500419 /* Do we need a layoutreturn? */
420 if (test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags))
421 pnfs_layoutreturn_before_put_lseg(lseg, lo, inode);
422
Fred Isamand684d2a2011-03-01 01:34:13 +0000423 if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
Trond Myklebustfaa4a542015-07-09 18:24:07 +0200424 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
425 spin_unlock(&inode->i_lock);
426 return;
427 }
Trond Myklebust8f0d27d2012-09-20 20:57:11 -0400428 pnfs_get_layout_hdr(lo);
Trond Myklebust4ef2e4f2015-02-05 17:27:39 -0500429 pnfs_layout_remove_lseg(lo, lseg);
430 spin_unlock(&inode->i_lock);
431 pnfs_free_lseg(lseg);
432 pnfs_put_layout_hdr(lo);
Fred Isaman4541d162011-01-06 11:36:23 +0000433 }
Andy Adamson974cec82010-10-20 00:18:02 -0400434}
Trond Myklebust9369a432012-09-18 20:57:08 -0400435EXPORT_SYMBOL_GPL(pnfs_put_lseg);
Andy Adamson974cec82010-10-20 00:18:02 -0400436
Trond Myklebust6543f802014-10-08 16:39:12 -0400437static void pnfs_free_lseg_async_work(struct work_struct *work)
Weston Andros Adamsone6cf82d2014-07-17 20:42:18 -0400438{
439 struct pnfs_layout_segment *lseg;
Trond Myklebust6543f802014-10-08 16:39:12 -0400440 struct pnfs_layout_hdr *lo;
Weston Andros Adamsone6cf82d2014-07-17 20:42:18 -0400441
442 lseg = container_of(work, struct pnfs_layout_segment, pls_work);
Trond Myklebust6543f802014-10-08 16:39:12 -0400443 lo = lseg->pls_layout;
Weston Andros Adamsone6cf82d2014-07-17 20:42:18 -0400444
Trond Myklebust6543f802014-10-08 16:39:12 -0400445 pnfs_free_lseg(lseg);
446 pnfs_put_layout_hdr(lo);
447}
448
449static void pnfs_free_lseg_async(struct pnfs_layout_segment *lseg)
450{
451 INIT_WORK(&lseg->pls_work, pnfs_free_lseg_async_work);
452 schedule_work(&lseg->pls_work);
Weston Andros Adamsone6cf82d2014-07-17 20:42:18 -0400453}
454
455void
Trond Myklebust6543f802014-10-08 16:39:12 -0400456pnfs_put_lseg_locked(struct pnfs_layout_segment *lseg)
Weston Andros Adamsone6cf82d2014-07-17 20:42:18 -0400457{
Trond Myklebust6543f802014-10-08 16:39:12 -0400458 if (!lseg)
459 return;
460
461 assert_spin_locked(&lseg->pls_layout->plh_inode->i_lock);
462
463 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
464 atomic_read(&lseg->pls_refcount),
465 test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
466 if (atomic_dec_and_test(&lseg->pls_refcount)) {
467 struct pnfs_layout_hdr *lo = lseg->pls_layout;
Trond Myklebustfaa4a542015-07-09 18:24:07 +0200468 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags))
469 return;
Trond Myklebust6543f802014-10-08 16:39:12 -0400470 pnfs_get_layout_hdr(lo);
471 pnfs_layout_remove_lseg(lo, lseg);
472 pnfs_free_lseg_async(lseg);
473 }
Weston Andros Adamsone6cf82d2014-07-17 20:42:18 -0400474}
Trond Myklebust6543f802014-10-08 16:39:12 -0400475EXPORT_SYMBOL_GPL(pnfs_put_lseg_locked);
Weston Andros Adamsone6cf82d2014-07-17 20:42:18 -0400476
Trond Myklebust3cb2df12013-06-03 11:24:36 -0400477static u64
Benny Halevyfb3296e2011-05-22 19:47:26 +0300478end_offset(u64 start, u64 len)
Fred Isaman4541d162011-01-06 11:36:23 +0000479{
Benny Halevyfb3296e2011-05-22 19:47:26 +0300480 u64 end;
481
482 end = start + len;
483 return end >= start ? end : NFS4_MAX_UINT64;
484}
485
Benny Halevyfb3296e2011-05-22 19:47:26 +0300486/*
487 * is l2 fully contained in l1?
488 * start1 end1
489 * [----------------------------------)
490 * start2 end2
491 * [----------------)
492 */
Trond Myklebust3cb2df12013-06-03 11:24:36 -0400493static bool
Trond Myklebust7dc0ac72013-06-03 11:30:24 -0400494pnfs_lseg_range_contained(const struct pnfs_layout_range *l1,
Trond Myklebust3cb2df12013-06-03 11:24:36 -0400495 const struct pnfs_layout_range *l2)
Benny Halevyfb3296e2011-05-22 19:47:26 +0300496{
497 u64 start1 = l1->offset;
498 u64 end1 = end_offset(start1, l1->length);
499 u64 start2 = l2->offset;
500 u64 end2 = end_offset(start2, l2->length);
501
502 return (start1 <= start2) && (end1 >= end2);
503}
504
505/*
506 * is l1 and l2 intersecting?
507 * start1 end1
508 * [----------------------------------)
509 * start2 end2
510 * [----------------)
511 */
Trond Myklebust3cb2df12013-06-03 11:24:36 -0400512static bool
Trond Myklebust7dc0ac72013-06-03 11:30:24 -0400513pnfs_lseg_range_intersecting(const struct pnfs_layout_range *l1,
Trond Myklebust3cb2df12013-06-03 11:24:36 -0400514 const struct pnfs_layout_range *l2)
Benny Halevyfb3296e2011-05-22 19:47:26 +0300515{
516 u64 start1 = l1->offset;
517 u64 end1 = end_offset(start1, l1->length);
518 u64 start2 = l2->offset;
519 u64 end2 = end_offset(start2, l2->length);
520
521 return (end1 == NFS4_MAX_UINT64 || end1 > start2) &&
522 (end2 == NFS4_MAX_UINT64 || end2 > start1);
523}
524
Fred Isaman4541d162011-01-06 11:36:23 +0000525static bool
Trond Myklebust3cb2df12013-06-03 11:24:36 -0400526should_free_lseg(const struct pnfs_layout_range *lseg_range,
527 const struct pnfs_layout_range *recall_range)
Fred Isaman4541d162011-01-06 11:36:23 +0000528{
Benny Halevy778b5502011-05-22 19:48:02 +0300529 return (recall_range->iomode == IOMODE_ANY ||
530 lseg_range->iomode == recall_range->iomode) &&
Trond Myklebust7dc0ac72013-06-03 11:30:24 -0400531 pnfs_lseg_range_intersecting(lseg_range, recall_range);
Fred Isaman4541d162011-01-06 11:36:23 +0000532}
533
Trond Myklebust24956802013-03-20 13:03:00 -0400534static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
535 struct list_head *tmp_list)
536{
537 if (!atomic_dec_and_test(&lseg->pls_refcount))
538 return false;
539 pnfs_layout_remove_lseg(lseg->pls_layout, lseg);
540 list_add(&lseg->pls_list, tmp_list);
541 return true;
542}
543
Fred Isaman4541d162011-01-06 11:36:23 +0000544/* Returns 1 if lseg is removed from list, 0 otherwise */
545static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
546 struct list_head *tmp_list)
547{
548 int rv = 0;
549
550 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
551 /* Remove the reference keeping the lseg in the
552 * list. It will now be removed when all
553 * outstanding io is finished.
554 */
Fred Isamand684d2a2011-03-01 01:34:13 +0000555 dprintk("%s: lseg %p ref %d\n", __func__, lseg,
556 atomic_read(&lseg->pls_refcount));
Trond Myklebust24956802013-03-20 13:03:00 -0400557 if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list))
Fred Isamand684d2a2011-03-01 01:34:13 +0000558 rv = 1;
Fred Isaman4541d162011-01-06 11:36:23 +0000559 }
560 return rv;
561}
562
563/* Returns count of number of matching invalid lsegs remaining in list
564 * after call.
565 */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000566int
Trond Myklebust49a85062012-09-18 20:43:31 -0400567pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
Fred Isaman4541d162011-01-06 11:36:23 +0000568 struct list_head *tmp_list,
Benny Halevy778b5502011-05-22 19:48:02 +0300569 struct pnfs_layout_range *recall_range)
Andy Adamson974cec82010-10-20 00:18:02 -0400570{
571 struct pnfs_layout_segment *lseg, *next;
Fred Isaman4541d162011-01-06 11:36:23 +0000572 int invalid = 0, removed = 0;
Andy Adamson974cec82010-10-20 00:18:02 -0400573
574 dprintk("%s:Begin lo %p\n", __func__, lo);
575
Trond Myklebust8006bfb2012-09-21 14:48:04 -0400576 if (list_empty(&lo->plh_segs))
Fred Isaman38511722011-02-03 18:28:50 +0000577 return 0;
Fred Isaman4541d162011-01-06 11:36:23 +0000578 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
Benny Halevy778b5502011-05-22 19:48:02 +0300579 if (!recall_range ||
580 should_free_lseg(&lseg->pls_range, recall_range)) {
Fred Isaman4541d162011-01-06 11:36:23 +0000581 dprintk("%s: freeing lseg %p iomode %d "
582 "offset %llu length %llu\n", __func__,
583 lseg, lseg->pls_range.iomode, lseg->pls_range.offset,
584 lseg->pls_range.length);
585 invalid++;
586 removed += mark_lseg_invalid(lseg, tmp_list);
587 }
588 dprintk("%s:Return %i\n", __func__, invalid - removed);
589 return invalid - removed;
Andy Adamson974cec82010-10-20 00:18:02 -0400590}
591
Fred Isamanf49f9ba2011-02-03 18:28:52 +0000592/* note free_me must contain lsegs from a single layout_hdr */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000593void
Fred Isaman4541d162011-01-06 11:36:23 +0000594pnfs_free_lseg_list(struct list_head *free_me)
Andy Adamson974cec82010-10-20 00:18:02 -0400595{
Fred Isaman4541d162011-01-06 11:36:23 +0000596 struct pnfs_layout_segment *lseg, *tmp;
Andy Adamson974cec82010-10-20 00:18:02 -0400597
Fred Isamanf49f9ba2011-02-03 18:28:52 +0000598 if (list_empty(free_me))
599 return;
600
Fred Isaman4541d162011-01-06 11:36:23 +0000601 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
Fred Isaman566052c2011-01-06 11:36:20 +0000602 list_del(&lseg->pls_list);
Trond Myklebust905ca192012-09-20 20:46:49 -0400603 pnfs_free_lseg(lseg);
Andy Adamson974cec82010-10-20 00:18:02 -0400604 }
605}
606
Benny Halevye5e94012010-10-20 00:18:01 -0400607void
608pnfs_destroy_layout(struct nfs_inode *nfsi)
609{
610 struct pnfs_layout_hdr *lo;
Andy Adamson974cec82010-10-20 00:18:02 -0400611 LIST_HEAD(tmp_list);
Benny Halevye5e94012010-10-20 00:18:01 -0400612
613 spin_lock(&nfsi->vfs_inode.i_lock);
614 lo = nfsi->layout;
615 if (lo) {
Fred Isaman38511722011-02-03 18:28:50 +0000616 lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
Trond Myklebust49a85062012-09-18 20:43:31 -0400617 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
Trond Myklebust3e621212012-09-24 13:07:16 -0400618 pnfs_get_layout_hdr(lo);
619 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
620 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
Peng Taoc8290132014-12-01 08:22:18 +0800621 pnfs_clear_retry_layoutget(lo);
Trond Myklebust3e621212012-09-24 13:07:16 -0400622 spin_unlock(&nfsi->vfs_inode.i_lock);
623 pnfs_free_lseg_list(&tmp_list);
624 pnfs_put_layout_hdr(lo);
625 } else
626 spin_unlock(&nfsi->vfs_inode.i_lock);
Benny Halevye5e94012010-10-20 00:18:01 -0400627}
Andy Adamson041245c2012-04-27 17:53:53 -0400628EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
Benny Halevye5e94012010-10-20 00:18:01 -0400629
Trond Myklebustfd9a8d72013-02-12 09:48:42 -0500630static bool
631pnfs_layout_add_bulk_destroy_list(struct inode *inode,
632 struct list_head *layout_list)
633{
634 struct pnfs_layout_hdr *lo;
635 bool ret = false;
636
637 spin_lock(&inode->i_lock);
638 lo = NFS_I(inode)->layout;
639 if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) {
640 pnfs_get_layout_hdr(lo);
641 list_add(&lo->plh_bulk_destroy, layout_list);
642 ret = true;
643 }
644 spin_unlock(&inode->i_lock);
645 return ret;
646}
647
648/* Caller must hold rcu_read_lock and clp->cl_lock */
649static int
650pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp,
651 struct nfs_server *server,
652 struct list_head *layout_list)
653{
654 struct pnfs_layout_hdr *lo, *next;
655 struct inode *inode;
656
657 list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) {
658 inode = igrab(lo->plh_inode);
659 if (inode == NULL)
660 continue;
661 list_del_init(&lo->plh_layouts);
662 if (pnfs_layout_add_bulk_destroy_list(inode, layout_list))
663 continue;
664 rcu_read_unlock();
665 spin_unlock(&clp->cl_lock);
666 iput(inode);
667 spin_lock(&clp->cl_lock);
668 rcu_read_lock();
669 return -EAGAIN;
670 }
671 return 0;
672}
673
674static int
675pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list,
676 bool is_bulk_recall)
677{
678 struct pnfs_layout_hdr *lo;
679 struct inode *inode;
680 struct pnfs_layout_range range = {
681 .iomode = IOMODE_ANY,
682 .offset = 0,
683 .length = NFS4_MAX_UINT64,
684 };
685 LIST_HEAD(lseg_list);
686 int ret = 0;
687
688 while (!list_empty(layout_list)) {
689 lo = list_entry(layout_list->next, struct pnfs_layout_hdr,
690 plh_bulk_destroy);
691 dprintk("%s freeing layout for inode %lu\n", __func__,
692 lo->plh_inode->i_ino);
693 inode = lo->plh_inode;
Christoph Hellwig7c5d1872014-09-10 08:23:29 -0700694
695 pnfs_layoutcommit_inode(inode, false);
696
Trond Myklebustfd9a8d72013-02-12 09:48:42 -0500697 spin_lock(&inode->i_lock);
698 list_del_init(&lo->plh_bulk_destroy);
699 lo->plh_block_lgets++; /* permanently block new LAYOUTGETs */
700 if (is_bulk_recall)
701 set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
702 if (pnfs_mark_matching_lsegs_invalid(lo, &lseg_list, &range))
703 ret = -EAGAIN;
704 spin_unlock(&inode->i_lock);
705 pnfs_free_lseg_list(&lseg_list);
706 pnfs_put_layout_hdr(lo);
707 iput(inode);
708 }
709 return ret;
710}
711
712int
713pnfs_destroy_layouts_byfsid(struct nfs_client *clp,
714 struct nfs_fsid *fsid,
715 bool is_recall)
716{
717 struct nfs_server *server;
718 LIST_HEAD(layout_list);
719
720 spin_lock(&clp->cl_lock);
721 rcu_read_lock();
722restart:
723 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
724 if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0)
725 continue;
726 if (pnfs_layout_bulk_destroy_byserver_locked(clp,
727 server,
728 &layout_list) != 0)
729 goto restart;
730 }
731 rcu_read_unlock();
732 spin_unlock(&clp->cl_lock);
733
734 if (list_empty(&layout_list))
735 return 0;
736 return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
737}
738
739int
740pnfs_destroy_layouts_byclid(struct nfs_client *clp,
741 bool is_recall)
742{
743 struct nfs_server *server;
744 LIST_HEAD(layout_list);
745
746 spin_lock(&clp->cl_lock);
747 rcu_read_lock();
748restart:
749 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
750 if (pnfs_layout_bulk_destroy_byserver_locked(clp,
751 server,
752 &layout_list) != 0)
753 goto restart;
754 }
755 rcu_read_unlock();
756 spin_unlock(&clp->cl_lock);
757
758 if (list_empty(&layout_list))
759 return 0;
760 return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
761}
762
Andy Adamson974cec82010-10-20 00:18:02 -0400763/*
764 * Called by the state manger to remove all layouts established under an
765 * expired lease.
766 */
767void
768pnfs_destroy_all_layouts(struct nfs_client *clp)
769{
Andy Adamsonc47abcf2011-06-15 17:52:40 -0400770 nfs4_deviceid_mark_client_invalid(clp);
771 nfs4_deviceid_purge_client(clp);
772
Trond Myklebustfd9a8d72013-02-12 09:48:42 -0500773 pnfs_destroy_layouts_byclid(clp, false);
Andy Adamson974cec82010-10-20 00:18:02 -0400774}
775
Trond Myklebust5a655032012-10-02 16:47:14 -0700776/*
777 * Compare 2 layout stateid sequence ids, to see which is newer,
778 * taking into account wraparound issues.
779 */
780static bool pnfs_seqid_is_newer(u32 s1, u32 s2)
781{
Trond Myklebust2c64c572014-02-12 09:36:59 -0500782 return (s32)(s1 - s2) > 0;
Trond Myklebust5a655032012-10-02 16:47:14 -0700783}
784
Fred Isamanfd6002e2011-01-06 11:36:22 +0000785/* update lo->plh_stateid with new if is more recent */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000786void
787pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
788 bool update_barrier)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400789{
Trond Myklebust22aaf712012-10-04 16:32:22 -0700790 u32 oldseq, newseq, new_barrier;
791 int empty = list_empty(&lo->plh_segs);
Andy Adamson974cec82010-10-20 00:18:02 -0400792
Trond Myklebust2d2f24a2012-03-04 18:13:57 -0500793 oldseq = be32_to_cpu(lo->plh_stateid.seqid);
794 newseq = be32_to_cpu(new->seqid);
Trond Myklebust22aaf712012-10-04 16:32:22 -0700795 if (empty || pnfs_seqid_is_newer(newseq, oldseq)) {
Trond Myklebustf597c532012-03-04 18:13:56 -0500796 nfs4_stateid_copy(&lo->plh_stateid, new);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000797 if (update_barrier) {
Trond Myklebust22aaf712012-10-04 16:32:22 -0700798 new_barrier = be32_to_cpu(new->seqid);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000799 } else {
800 /* Because of wraparound, we want to keep the barrier
Trond Myklebust22aaf712012-10-04 16:32:22 -0700801 * "close" to the current seqids.
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000802 */
Trond Myklebust22aaf712012-10-04 16:32:22 -0700803 new_barrier = newseq - atomic_read(&lo->plh_outstanding);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000804 }
Trond Myklebust22aaf712012-10-04 16:32:22 -0700805 if (empty || pnfs_seqid_is_newer(new_barrier, lo->plh_barrier))
806 lo->plh_barrier = new_barrier;
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000807 }
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400808}
809
Trond Myklebust19c54ab2012-10-05 16:56:58 -0700810static bool
811pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo,
812 const nfs4_stateid *stateid)
813{
814 u32 seqid = be32_to_cpu(stateid->seqid);
815
816 return !pnfs_seqid_is_newer(seqid, lo->plh_barrier);
817}
818
Fred Isamancf7d63f2011-01-06 11:36:25 +0000819/* lget is set to 1 if called from inside send_layoutget call chain */
820static bool
Trond Myklebuste1c06f82015-08-04 16:40:08 -0400821pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo)
Fred Isamancf7d63f2011-01-06 11:36:25 +0000822{
Fred Isamanf7e89172011-01-06 11:36:32 +0000823 return lo->plh_block_lgets ||
Trond Myklebuste1c06f82015-08-04 16:40:08 -0400824 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
Fred Isamancf7d63f2011-01-06 11:36:25 +0000825}
826
Fred Isamanfd6002e2011-01-06 11:36:22 +0000827int
828pnfs_choose_layoutget_stateid(nfs4_stateid *dst, struct pnfs_layout_hdr *lo,
Peng Taoce6ab4f2014-09-06 00:53:24 +0800829 struct pnfs_layout_range *range,
Fred Isamanfd6002e2011-01-06 11:36:22 +0000830 struct nfs4_state *open_state)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400831{
Fred Isamanfd6002e2011-01-06 11:36:22 +0000832 int status = 0;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400833
834 dprintk("--> %s\n", __func__);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000835 spin_lock(&lo->plh_inode->i_lock);
Trond Myklebuste1c06f82015-08-04 16:40:08 -0400836 if (pnfs_layoutgets_blocked(lo)) {
Fred Isamancf7d63f2011-01-06 11:36:25 +0000837 status = -EAGAIN;
Trond Myklebust5d422302013-03-14 16:57:48 -0400838 } else if (!nfs4_valid_open_stateid(open_state)) {
839 status = -EBADF;
Christoph Hellwig47abade2014-08-21 11:09:22 -0500840 } else if (list_empty(&lo->plh_segs) ||
841 test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) {
Fred Isamanfd6002e2011-01-06 11:36:22 +0000842 int seq;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400843
Fred Isamanfd6002e2011-01-06 11:36:22 +0000844 do {
845 seq = read_seqbegin(&open_state->seqlock);
Trond Myklebustf597c532012-03-04 18:13:56 -0500846 nfs4_stateid_copy(dst, &open_state->stateid);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000847 } while (read_seqretry(&open_state->seqlock, seq));
848 } else
Trond Myklebustf597c532012-03-04 18:13:56 -0500849 nfs4_stateid_copy(dst, &lo->plh_stateid);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000850 spin_unlock(&lo->plh_inode->i_lock);
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400851 dprintk("<-- %s\n", __func__);
Fred Isamanfd6002e2011-01-06 11:36:22 +0000852 return status;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400853}
854
855/*
856* Get layout from server.
857* for now, assume that whole file layouts are requested.
858* arg->offset: 0
859* arg->length: all ones
860*/
Benny Halevye5e94012010-10-20 00:18:01 -0400861static struct pnfs_layout_segment *
862send_layoutget(struct pnfs_layout_hdr *lo,
863 struct nfs_open_context *ctx,
Benny Halevyfb3296e2011-05-22 19:47:26 +0300864 struct pnfs_layout_range *range,
Trond Myklebusta75b9df2011-05-11 18:00:51 -0400865 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -0400866{
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000867 struct inode *ino = lo->plh_inode;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400868 struct nfs_server *server = NFS_SERVER(ino);
869 struct nfs4_layoutget *lgp;
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -0400870 struct pnfs_layout_segment *lseg;
Trond Myklebust2d89a1d2015-08-31 02:05:47 -0700871 loff_t i_size;
Benny Halevye5e94012010-10-20 00:18:01 -0400872
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400873 dprintk("--> %s\n", __func__);
874
Jeff Layton4f2e9dc2015-11-25 13:43:14 -0500875 /*
876 * Synchronously retrieve layout information from server and
877 * store in lseg. If we race with a concurrent seqid morphing
878 * op, then re-send the LAYOUTGET.
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400879 */
Jeff Layton4f2e9dc2015-11-25 13:43:14 -0500880 do {
881 lgp = kzalloc(sizeof(*lgp), gfp_flags);
882 if (lgp == NULL)
883 return NULL;
884
885 i_size = i_size_read(ino);
886
887 lgp->args.minlength = PAGE_CACHE_SIZE;
888 if (lgp->args.minlength > range->length)
889 lgp->args.minlength = range->length;
890 if (range->iomode == IOMODE_READ) {
891 if (range->offset >= i_size)
892 lgp->args.minlength = 0;
893 else if (i_size - range->offset < lgp->args.minlength)
894 lgp->args.minlength = i_size - range->offset;
895 }
896 lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
897 lgp->args.range = *range;
898 lgp->args.type = server->pnfs_curr_ld->id;
899 lgp->args.inode = ino;
900 lgp->args.ctx = get_nfs_open_context(ctx);
901 lgp->gfp_flags = gfp_flags;
902 lgp->cred = lo->plh_lc_cred;
903
904 lseg = nfs4_proc_layoutget(lgp, gfp_flags);
905 } while (lseg == ERR_PTR(-EAGAIN));
906
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -0400907 if (IS_ERR(lseg)) {
908 switch (PTR_ERR(lseg)) {
909 case -ENOMEM:
910 case -ERESTARTSYS:
911 break;
912 default:
913 /* remember that LAYOUTGET failed and suspend trying */
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400914 pnfs_layout_io_set_failed(lo, range->iomode);
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -0400915 }
916 return NULL;
Tom Haynesd67ae822014-12-11 17:02:04 -0500917 } else
918 pnfs_layout_clear_fail_bit(lo,
919 pnfs_iomode_to_fail_bit(range->iomode));
Weston Andros Adamson35124a02011-03-24 16:48:21 -0400920
Andy Adamson974cec82010-10-20 00:18:02 -0400921 return lseg;
922}
923
Trond Myklebust24956802013-03-20 13:03:00 -0400924static void pnfs_clear_layoutcommit(struct inode *inode,
925 struct list_head *head)
926{
927 struct nfs_inode *nfsi = NFS_I(inode);
928 struct pnfs_layout_segment *lseg, *tmp;
929
930 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
931 return;
932 list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) {
933 if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
934 continue;
935 pnfs_lseg_dec_and_remove_zero(lseg, head);
936 }
937}
938
Tom Haynesd67ae822014-12-11 17:02:04 -0500939void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo)
940{
941 clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags);
942 smp_mb__after_atomic();
943 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN);
Trond Myklebust7f273922015-07-09 18:40:01 +0200944 rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq);
Tom Haynesd67ae822014-12-11 17:02:04 -0500945}
946
Peng Taof40eb5d2014-09-06 00:53:22 +0800947static int
948pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, nfs4_stateid stateid,
Peng Tao6c166052014-11-17 09:30:40 +0800949 enum pnfs_iomode iomode, bool sync)
Peng Taof40eb5d2014-09-06 00:53:22 +0800950{
951 struct inode *ino = lo->plh_inode;
952 struct nfs4_layoutreturn *lrp;
953 int status = 0;
954
Trond Myklebuste4af4402015-02-05 17:05:08 -0500955 lrp = kzalloc(sizeof(*lrp), GFP_NOFS);
Peng Taof40eb5d2014-09-06 00:53:22 +0800956 if (unlikely(lrp == NULL)) {
957 status = -ENOMEM;
958 spin_lock(&ino->i_lock);
Tom Haynesd67ae822014-12-11 17:02:04 -0500959 pnfs_clear_layoutreturn_waitbit(lo);
Peng Taof40eb5d2014-09-06 00:53:22 +0800960 spin_unlock(&ino->i_lock);
961 pnfs_put_layout_hdr(lo);
962 goto out;
963 }
964
965 lrp->args.stateid = stateid;
966 lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
967 lrp->args.inode = ino;
Peng Tao15eb67c2014-11-17 09:30:36 +0800968 lrp->args.range.iomode = iomode;
969 lrp->args.range.offset = 0;
970 lrp->args.range.length = NFS4_MAX_UINT64;
Peng Taof40eb5d2014-09-06 00:53:22 +0800971 lrp->args.layout = lo;
972 lrp->clp = NFS_SERVER(ino)->nfs_client;
973 lrp->cred = lo->plh_lc_cred;
974
Peng Tao6c166052014-11-17 09:30:40 +0800975 status = nfs4_proc_layoutreturn(lrp, sync);
Peng Taof40eb5d2014-09-06 00:53:22 +0800976out:
977 dprintk("<-- %s status: %d\n", __func__, status);
978 return status;
979}
980
Andy Adamson293b3b02012-06-20 15:03:34 -0400981/*
982 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
983 * when the layout segment list is empty.
984 *
985 * Note that a pnfs_layout_hdr can exist with an empty layout segment
986 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the
987 * deviceid is marked invalid.
988 */
Benny Halevycbe82602011-05-22 19:52:37 +0300989int
990_pnfs_return_layout(struct inode *ino)
991{
992 struct pnfs_layout_hdr *lo = NULL;
993 struct nfs_inode *nfsi = NFS_I(ino);
994 LIST_HEAD(tmp_list);
Benny Halevycbe82602011-05-22 19:52:37 +0300995 nfs4_stateid stateid;
Andy Adamson293b3b02012-06-20 15:03:34 -0400996 int status = 0, empty;
Trond Myklebust7f273922015-07-09 18:40:01 +0200997 bool send;
Benny Halevycbe82602011-05-22 19:52:37 +0300998
Andy Adamson366d5052012-06-20 15:03:33 -0400999 dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
Benny Halevycbe82602011-05-22 19:52:37 +03001000
1001 spin_lock(&ino->i_lock);
1002 lo = nfsi->layout;
Trond Myklebuste5929f32012-09-21 16:37:02 -04001003 if (!lo) {
Benny Halevycbe82602011-05-22 19:52:37 +03001004 spin_unlock(&ino->i_lock);
Andy Adamson293b3b02012-06-20 15:03:34 -04001005 dprintk("NFS: %s no layout to return\n", __func__);
1006 goto out;
Benny Halevycbe82602011-05-22 19:52:37 +03001007 }
1008 stateid = nfsi->layout->plh_stateid;
1009 /* Reference matched in nfs4_layoutreturn_release */
Trond Myklebust70c3bd22012-09-18 20:51:13 -04001010 pnfs_get_layout_hdr(lo);
Andy Adamson293b3b02012-06-20 15:03:34 -04001011 empty = list_empty(&lo->plh_segs);
Trond Myklebust24956802013-03-20 13:03:00 -04001012 pnfs_clear_layoutcommit(ino, &tmp_list);
Trond Myklebust49a85062012-09-18 20:43:31 -04001013 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL);
Christoph Hellwigc88953d2014-09-10 08:23:31 -07001014
1015 if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) {
1016 struct pnfs_layout_range range = {
1017 .iomode = IOMODE_ANY,
1018 .offset = 0,
1019 .length = NFS4_MAX_UINT64,
1020 };
1021 NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range);
1022 }
1023
Andy Adamson293b3b02012-06-20 15:03:34 -04001024 /* Don't send a LAYOUTRETURN if list was initially empty */
1025 if (empty) {
1026 spin_unlock(&ino->i_lock);
Andy Adamson293b3b02012-06-20 15:03:34 -04001027 dprintk("NFS: %s no layout segments to return\n", __func__);
Trond Myklebust7f273922015-07-09 18:40:01 +02001028 goto out_put_layout_hdr;
Andy Adamson293b3b02012-06-20 15:03:34 -04001029 }
Christoph Hellwig47abade2014-08-21 11:09:22 -05001030
1031 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
Trond Myklebust7f273922015-07-09 18:40:01 +02001032 send = pnfs_prepare_layoutreturn(lo);
Benny Halevycbe82602011-05-22 19:52:37 +03001033 spin_unlock(&ino->i_lock);
1034 pnfs_free_lseg_list(&tmp_list);
Trond Myklebust7f273922015-07-09 18:40:01 +02001035 if (send)
1036 status = pnfs_send_layoutreturn(lo, stateid, IOMODE_ANY, true);
1037out_put_layout_hdr:
1038 pnfs_put_layout_hdr(lo);
Benny Halevycbe82602011-05-22 19:52:37 +03001039out:
1040 dprintk("<-- %s status: %d\n", __func__, status);
1041 return status;
1042}
Andy Adamson0a57cda2012-04-27 17:53:50 -04001043EXPORT_SYMBOL_GPL(_pnfs_return_layout);
Benny Halevycbe82602011-05-22 19:52:37 +03001044
Trond Myklebust24028672013-03-20 13:23:33 -04001045int
1046pnfs_commit_and_return_layout(struct inode *inode)
1047{
1048 struct pnfs_layout_hdr *lo;
1049 int ret;
1050
1051 spin_lock(&inode->i_lock);
1052 lo = NFS_I(inode)->layout;
1053 if (lo == NULL) {
1054 spin_unlock(&inode->i_lock);
1055 return 0;
1056 }
1057 pnfs_get_layout_hdr(lo);
1058 /* Block new layoutgets and read/write to ds */
1059 lo->plh_block_lgets++;
1060 spin_unlock(&inode->i_lock);
1061 filemap_fdatawait(inode->i_mapping);
1062 ret = pnfs_layoutcommit_inode(inode, true);
1063 if (ret == 0)
1064 ret = _pnfs_return_layout(inode);
1065 spin_lock(&inode->i_lock);
1066 lo->plh_block_lgets--;
1067 spin_unlock(&inode->i_lock);
1068 pnfs_put_layout_hdr(lo);
1069 return ret;
1070}
1071
Fred Isamanf7e89172011-01-06 11:36:32 +00001072bool pnfs_roc(struct inode *ino)
1073{
Trond Myklebust40dd4b72015-01-24 13:54:37 -05001074 struct nfs_inode *nfsi = NFS_I(ino);
1075 struct nfs_open_context *ctx;
1076 struct nfs4_state *state;
Fred Isamanf7e89172011-01-06 11:36:32 +00001077 struct pnfs_layout_hdr *lo;
1078 struct pnfs_layout_segment *lseg, *tmp;
Peng Tao193e3aa2014-11-17 09:30:41 +08001079 nfs4_stateid stateid;
Fred Isamanf7e89172011-01-06 11:36:32 +00001080 LIST_HEAD(tmp_list);
Peng Taoe755d632015-08-19 13:49:19 +08001081 bool found = false, layoutreturn = false, roc = false;
Fred Isamanf7e89172011-01-06 11:36:32 +00001082
1083 spin_lock(&ino->i_lock);
Trond Myklebust40dd4b72015-01-24 13:54:37 -05001084 lo = nfsi->layout;
Peng Tao39761432015-08-21 12:49:44 +08001085 if (!lo || test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags))
Trond Myklebust40dd4b72015-01-24 13:54:37 -05001086 goto out_noroc;
1087
Peng Taoe755d632015-08-19 13:49:19 +08001088 /* no roc if we hold a delegation */
Trond Myklebust40dd4b72015-01-24 13:54:37 -05001089 if (nfs4_check_delegation(ino, FMODE_READ))
1090 goto out_noroc;
1091
1092 list_for_each_entry(ctx, &nfsi->open_files, list) {
1093 state = ctx->state;
1094 /* Don't return layout if there is open file state */
1095 if (state != NULL && state->state != 0)
1096 goto out_noroc;
1097 }
1098
Peng Taoe755d632015-08-19 13:49:19 +08001099 stateid = lo->plh_stateid;
1100 /* always send layoutreturn if being marked so */
1101 if (test_and_clear_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE,
1102 &lo->plh_flags))
1103 layoutreturn = pnfs_prepare_layoutreturn(lo);
1104
Peng Taoc8290132014-12-01 08:22:18 +08001105 pnfs_clear_retry_layoutget(lo);
Fred Isamanf7e89172011-01-06 11:36:32 +00001106 list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list)
Peng Taoe755d632015-08-19 13:49:19 +08001107 /* If we are sending layoutreturn, invalidate all valid lsegs */
1108 if (layoutreturn || test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
Fred Isamanf7e89172011-01-06 11:36:32 +00001109 mark_lseg_invalid(lseg, &tmp_list);
1110 found = true;
1111 }
Peng Tao500d7012015-09-22 11:35:22 +08001112 /* ROC in two conditions:
Peng Taoe755d632015-08-19 13:49:19 +08001113 * 1. there are ROC lsegs
1114 * 2. we don't send layoutreturn
Peng Taoe755d632015-08-19 13:49:19 +08001115 */
Peng Tao500d7012015-09-22 11:35:22 +08001116 if (found && !layoutreturn) {
1117 /* lo ref dropped in pnfs_roc_release() */
1118 pnfs_get_layout_hdr(lo);
Peng Taoe755d632015-08-19 13:49:19 +08001119 roc = true;
Peng Tao500d7012015-09-22 11:35:22 +08001120 }
Peng Taoe755d632015-08-19 13:49:19 +08001121
1122out_noroc:
Fred Isamanf7e89172011-01-06 11:36:32 +00001123 spin_unlock(&ino->i_lock);
1124 pnfs_free_lseg_list(&tmp_list);
Trond Myklebust71401712015-03-25 12:36:13 -04001125 pnfs_layoutcommit_inode(ino, true);
Peng Taoe755d632015-08-19 13:49:19 +08001126 if (layoutreturn)
Peng Tao27b6f532014-10-20 14:44:38 +08001127 pnfs_send_layoutreturn(lo, stateid, IOMODE_ANY, true);
Peng Taoe755d632015-08-19 13:49:19 +08001128 return roc;
Fred Isamanf7e89172011-01-06 11:36:32 +00001129}
1130
1131void pnfs_roc_release(struct inode *ino)
1132{
1133 struct pnfs_layout_hdr *lo;
1134
1135 spin_lock(&ino->i_lock);
1136 lo = NFS_I(ino)->layout;
Trond Myklebust5c4a79f2015-08-04 15:57:13 -04001137 pnfs_clear_layoutreturn_waitbit(lo);
Trond Myklebust6622c3e2012-09-20 17:23:11 -04001138 if (atomic_dec_and_test(&lo->plh_refcount)) {
1139 pnfs_detach_layout_hdr(lo);
1140 spin_unlock(&ino->i_lock);
1141 pnfs_free_layout_hdr(lo);
1142 } else
1143 spin_unlock(&ino->i_lock);
Fred Isamanf7e89172011-01-06 11:36:32 +00001144}
1145
1146void pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
1147{
1148 struct pnfs_layout_hdr *lo;
1149
1150 spin_lock(&ino->i_lock);
1151 lo = NFS_I(ino)->layout;
Trond Myklebust0f35ad62012-10-04 16:28:17 -07001152 if (pnfs_seqid_is_newer(barrier, lo->plh_barrier))
Fred Isamanf7e89172011-01-06 11:36:32 +00001153 lo->plh_barrier = barrier;
1154 spin_unlock(&ino->i_lock);
Trond Myklebust6a463beb2015-08-20 15:40:47 -05001155 trace_nfs4_layoutreturn_on_close(ino, 0);
Fred Isamanf7e89172011-01-06 11:36:32 +00001156}
1157
Trond Myklebust4ff376f2015-08-18 23:23:21 -05001158void pnfs_roc_get_barrier(struct inode *ino, u32 *barrier)
Fred Isamanf7e89172011-01-06 11:36:32 +00001159{
1160 struct nfs_inode *nfsi = NFS_I(ino);
Trond Myklebust7fdab062012-09-20 20:15:57 -04001161 struct pnfs_layout_hdr *lo;
Trond Myklebust7fdab062012-09-20 20:15:57 -04001162 u32 current_seqid;
Fred Isamanf7e89172011-01-06 11:36:32 +00001163
1164 spin_lock(&ino->i_lock);
Trond Myklebust7fdab062012-09-20 20:15:57 -04001165 lo = nfsi->layout;
1166 current_seqid = be32_to_cpu(lo->plh_stateid.seqid);
Fred Isamanf7e89172011-01-06 11:36:32 +00001167
Trond Myklebust7fdab062012-09-20 20:15:57 -04001168 /* Since close does not return a layout stateid for use as
1169 * a barrier, we choose the worst-case barrier.
1170 */
1171 *barrier = current_seqid + atomic_read(&lo->plh_outstanding);
Fred Isamanf7e89172011-01-06 11:36:32 +00001172 spin_unlock(&ino->i_lock);
Fred Isamanf7e89172011-01-06 11:36:32 +00001173}
1174
Peng Tao500d7012015-09-22 11:35:22 +08001175bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task)
1176{
1177 struct nfs_inode *nfsi = NFS_I(ino);
1178 struct pnfs_layout_hdr *lo;
1179 bool sleep = false;
1180
1181 /* we might not have grabbed lo reference. so need to check under
1182 * i_lock */
1183 spin_lock(&ino->i_lock);
1184 lo = nfsi->layout;
1185 if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags))
1186 sleep = true;
1187 spin_unlock(&ino->i_lock);
1188
1189 if (sleep)
1190 rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
1191
1192 return sleep;
1193}
1194
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001195/*
1196 * Compare two layout segments for sorting into layout cache.
1197 * We want to preferentially return RW over RO layouts, so ensure those
1198 * are seen first.
1199 */
1200static s64
Trond Myklebust7dc0ac72013-06-03 11:30:24 -04001201pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1,
Trond Myklebust3cb2df12013-06-03 11:24:36 -04001202 const struct pnfs_layout_range *l2)
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001203{
Benny Halevyfb3296e2011-05-22 19:47:26 +03001204 s64 d;
1205
1206 /* high offset > low offset */
1207 d = l1->offset - l2->offset;
1208 if (d)
1209 return d;
1210
1211 /* short length > long length */
1212 d = l2->length - l1->length;
1213 if (d)
1214 return d;
1215
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001216 /* read > read/write */
Benny Halevyfb3296e2011-05-22 19:47:26 +03001217 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001218}
1219
Trond Myklebust03772d22015-08-25 08:54:17 -04001220static bool
1221pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1,
1222 const struct pnfs_layout_range *l2)
Andy Adamson974cec82010-10-20 00:18:02 -04001223{
Trond Myklebust03772d22015-08-25 08:54:17 -04001224 return pnfs_lseg_range_cmp(l1, l2) > 0;
1225}
1226
1227static bool
1228pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg,
1229 struct pnfs_layout_segment *old)
1230{
1231 return false;
1232}
1233
1234void
1235pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo,
1236 struct pnfs_layout_segment *lseg,
1237 bool (*is_after)(const struct pnfs_layout_range *,
1238 const struct pnfs_layout_range *),
1239 bool (*do_merge)(struct pnfs_layout_segment *,
1240 struct pnfs_layout_segment *),
1241 struct list_head *free_me)
1242{
1243 struct pnfs_layout_segment *lp, *tmp;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001244
Andy Adamson974cec82010-10-20 00:18:02 -04001245 dprintk("%s:Begin\n", __func__);
1246
Trond Myklebust03772d22015-08-25 08:54:17 -04001247 list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) {
1248 if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0)
1249 continue;
1250 if (do_merge(lseg, lp)) {
1251 mark_lseg_invalid(lp, free_me);
1252 continue;
1253 }
1254 if (is_after(&lseg->pls_range, &lp->pls_range))
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001255 continue;
Fred Isaman566052c2011-01-06 11:36:20 +00001256 list_add_tail(&lseg->pls_list, &lp->pls_list);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001257 dprintk("%s: inserted lseg %p "
1258 "iomode %d offset %llu length %llu before "
1259 "lp %p iomode %d offset %llu length %llu\n",
Fred Isaman566052c2011-01-06 11:36:20 +00001260 __func__, lseg, lseg->pls_range.iomode,
1261 lseg->pls_range.offset, lseg->pls_range.length,
1262 lp, lp->pls_range.iomode, lp->pls_range.offset,
1263 lp->pls_range.length);
Benny Halevyfb3296e2011-05-22 19:47:26 +03001264 goto out;
Andy Adamson974cec82010-10-20 00:18:02 -04001265 }
Benny Halevyfb3296e2011-05-22 19:47:26 +03001266 list_add_tail(&lseg->pls_list, &lo->plh_segs);
1267 dprintk("%s: inserted lseg %p "
1268 "iomode %d offset %llu length %llu at tail\n",
1269 __func__, lseg, lseg->pls_range.iomode,
1270 lseg->pls_range.offset, lseg->pls_range.length);
1271out:
Trond Myklebust70c3bd22012-09-18 20:51:13 -04001272 pnfs_get_layout_hdr(lo);
Andy Adamson974cec82010-10-20 00:18:02 -04001273
1274 dprintk("%s:Return\n", __func__);
Benny Halevye5e94012010-10-20 00:18:01 -04001275}
Trond Myklebust03772d22015-08-25 08:54:17 -04001276EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg);
1277
1278static void
1279pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,
1280 struct pnfs_layout_segment *lseg,
1281 struct list_head *free_me)
1282{
1283 struct inode *inode = lo->plh_inode;
1284 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
1285
1286 if (ld->add_lseg != NULL)
1287 ld->add_lseg(lo, lseg, free_me);
1288 else
1289 pnfs_generic_layout_insert_lseg(lo, lseg,
1290 pnfs_lseg_range_is_after,
1291 pnfs_lseg_no_merge,
1292 free_me);
1293}
Benny Halevye5e94012010-10-20 00:18:01 -04001294
1295static struct pnfs_layout_hdr *
Peng Tao9fa40752011-07-30 20:52:32 -04001296alloc_init_layout_hdr(struct inode *ino,
1297 struct nfs_open_context *ctx,
1298 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -04001299{
1300 struct pnfs_layout_hdr *lo;
1301
Benny Halevy636fb9c2011-05-22 19:51:33 +03001302 lo = pnfs_alloc_layout_hdr(ino, gfp_flags);
Benny Halevye5e94012010-10-20 00:18:01 -04001303 if (!lo)
1304 return NULL;
Fred Isamancc6e5342011-01-06 11:36:28 +00001305 atomic_set(&lo->plh_refcount, 1);
Fred Isamanb7edfaa2011-01-06 11:36:21 +00001306 INIT_LIST_HEAD(&lo->plh_layouts);
1307 INIT_LIST_HEAD(&lo->plh_segs);
Trond Myklebustfd9a8d72013-02-12 09:48:42 -05001308 INIT_LIST_HEAD(&lo->plh_bulk_destroy);
Fred Isamanb7edfaa2011-01-06 11:36:21 +00001309 lo->plh_inode = ino;
Trond Myklebust5cc22162013-05-21 09:26:49 -04001310 lo->plh_lc_cred = get_rpccred(ctx->cred);
Benny Halevye5e94012010-10-20 00:18:01 -04001311 return lo;
1312}
1313
1314static struct pnfs_layout_hdr *
Peng Tao9fa40752011-07-30 20:52:32 -04001315pnfs_find_alloc_layout(struct inode *ino,
1316 struct nfs_open_context *ctx,
1317 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -04001318{
1319 struct nfs_inode *nfsi = NFS_I(ino);
1320 struct pnfs_layout_hdr *new = NULL;
1321
1322 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);
1323
Trond Myklebust251ec412012-10-02 15:41:05 -07001324 if (nfsi->layout != NULL)
1325 goto out_existing;
Benny Halevye5e94012010-10-20 00:18:01 -04001326 spin_unlock(&ino->i_lock);
Peng Tao9fa40752011-07-30 20:52:32 -04001327 new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
Benny Halevye5e94012010-10-20 00:18:01 -04001328 spin_lock(&ino->i_lock);
1329
Trond Myklebust251ec412012-10-02 15:41:05 -07001330 if (likely(nfsi->layout == NULL)) { /* Won the race? */
Benny Halevye5e94012010-10-20 00:18:01 -04001331 nfsi->layout = new;
Trond Myklebust251ec412012-10-02 15:41:05 -07001332 return new;
Yanchuan Nian7175fe92012-10-31 16:05:48 +08001333 } else if (new != NULL)
1334 pnfs_free_layout_hdr(new);
Trond Myklebust251ec412012-10-02 15:41:05 -07001335out_existing:
1336 pnfs_get_layout_hdr(nfsi->layout);
Benny Halevye5e94012010-10-20 00:18:01 -04001337 return nfsi->layout;
1338}
1339
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001340/*
1341 * iomode matching rules:
1342 * iomode lseg match
1343 * ----- ----- -----
1344 * ANY READ true
1345 * ANY RW true
1346 * RW READ false
1347 * RW RW true
1348 * READ READ true
1349 * READ RW true
1350 */
Trond Myklebust3cb2df12013-06-03 11:24:36 -04001351static bool
Trond Myklebust7dc0ac72013-06-03 11:30:24 -04001352pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range,
Trond Myklebust3cb2df12013-06-03 11:24:36 -04001353 const struct pnfs_layout_range *range)
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001354{
Benny Halevyfb3296e2011-05-22 19:47:26 +03001355 struct pnfs_layout_range range1;
1356
1357 if ((range->iomode == IOMODE_RW &&
1358 ls_range->iomode != IOMODE_RW) ||
Trond Myklebust7dc0ac72013-06-03 11:30:24 -04001359 !pnfs_lseg_range_intersecting(ls_range, range))
Benny Halevyfb3296e2011-05-22 19:47:26 +03001360 return 0;
1361
1362 /* range1 covers only the first byte in the range */
1363 range1 = *range;
1364 range1.length = 1;
Trond Myklebust7dc0ac72013-06-03 11:30:24 -04001365 return pnfs_lseg_range_contained(ls_range, &range1);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001366}
1367
1368/*
1369 * lookup range in layout
1370 */
Benny Halevye5e94012010-10-20 00:18:01 -04001371static struct pnfs_layout_segment *
Benny Halevyfb3296e2011-05-22 19:47:26 +03001372pnfs_find_lseg(struct pnfs_layout_hdr *lo,
1373 struct pnfs_layout_range *range)
Benny Halevye5e94012010-10-20 00:18:01 -04001374{
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001375 struct pnfs_layout_segment *lseg, *ret = NULL;
1376
1377 dprintk("%s:Begin\n", __func__);
1378
Fred Isamanb7edfaa2011-01-06 11:36:21 +00001379 list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
Fred Isaman4541d162011-01-06 11:36:23 +00001380 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
Peng Taoce6ab4f2014-09-06 00:53:24 +08001381 !test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
Trond Myklebust7dc0ac72013-06-03 11:30:24 -04001382 pnfs_lseg_range_match(&lseg->pls_range, range)) {
Trond Myklebust9369a432012-09-18 20:57:08 -04001383 ret = pnfs_get_lseg(lseg);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001384 break;
1385 }
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001386 }
1387
1388 dprintk("%s:Return lseg %p ref %d\n",
Fred Isaman4541d162011-01-06 11:36:23 +00001389 __func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001390 return ret;
Benny Halevye5e94012010-10-20 00:18:01 -04001391}
1392
1393/*
Andy Adamsond23d61c2012-05-23 05:02:37 -04001394 * Use mdsthreshold hints set at each OPEN to determine if I/O should go
1395 * to the MDS or over pNFS
1396 *
1397 * The nfs_inode read_io and write_io fields are cumulative counters reset
1398 * when there are no layout segments. Note that in pnfs_update_layout iomode
1399 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a
1400 * WRITE request.
1401 *
1402 * A return of true means use MDS I/O.
1403 *
1404 * From rfc 5661:
1405 * If a file's size is smaller than the file size threshold, data accesses
1406 * SHOULD be sent to the metadata server. If an I/O request has a length that
1407 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata
1408 * server. If both file size and I/O size are provided, the client SHOULD
1409 * reach or exceed both thresholds before sending its read or write
1410 * requests to the data server.
1411 */
1412static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
1413 struct inode *ino, int iomode)
1414{
1415 struct nfs4_threshold *t = ctx->mdsthreshold;
1416 struct nfs_inode *nfsi = NFS_I(ino);
1417 loff_t fsize = i_size_read(ino);
1418 bool size = false, size_set = false, io = false, io_set = false, ret = false;
1419
1420 if (t == NULL)
1421 return ret;
1422
1423 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n",
1424 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz);
1425
1426 switch (iomode) {
1427 case IOMODE_READ:
1428 if (t->bm & THRESHOLD_RD) {
1429 dprintk("%s fsize %llu\n", __func__, fsize);
1430 size_set = true;
1431 if (fsize < t->rd_sz)
1432 size = true;
1433 }
1434 if (t->bm & THRESHOLD_RD_IO) {
1435 dprintk("%s nfsi->read_io %llu\n", __func__,
1436 nfsi->read_io);
1437 io_set = true;
1438 if (nfsi->read_io < t->rd_io_sz)
1439 io = true;
1440 }
1441 break;
1442 case IOMODE_RW:
1443 if (t->bm & THRESHOLD_WR) {
1444 dprintk("%s fsize %llu\n", __func__, fsize);
1445 size_set = true;
1446 if (fsize < t->wr_sz)
1447 size = true;
1448 }
1449 if (t->bm & THRESHOLD_WR_IO) {
1450 dprintk("%s nfsi->write_io %llu\n", __func__,
1451 nfsi->write_io);
1452 io_set = true;
1453 if (nfsi->write_io < t->wr_io_sz)
1454 io = true;
1455 }
1456 break;
1457 }
1458 if (size_set && io_set) {
1459 if (size && io)
1460 ret = true;
1461 } else if (size || io)
1462 ret = true;
1463
1464 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret);
1465 return ret;
1466}
1467
Peng Taoaa8a45e2014-12-01 08:22:23 +08001468/* stop waiting if someone clears NFS_LAYOUT_RETRY_LAYOUTGET bit. */
1469static int pnfs_layoutget_retry_bit_wait(struct wait_bit_key *key)
1470{
1471 if (!test_bit(NFS_LAYOUT_RETRY_LAYOUTGET, key->flags))
1472 return 1;
1473 return nfs_wait_bit_killable(key);
1474}
1475
1476static bool pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo)
1477{
Trond Myklebust8f70f532015-08-04 16:09:44 -04001478 if (!pnfs_should_retry_layoutget(lo))
1479 return false;
Peng Taoaa8a45e2014-12-01 08:22:23 +08001480 /*
1481 * send layoutcommit as it can hold up layoutreturn due to lseg
1482 * reference
1483 */
1484 pnfs_layoutcommit_inode(lo->plh_inode, false);
1485 return !wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN,
1486 pnfs_layoutget_retry_bit_wait,
1487 TASK_UNINTERRUPTIBLE);
1488}
1489
Tom Haynesd67ae822014-12-11 17:02:04 -05001490static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo)
1491{
1492 unsigned long *bitlock = &lo->plh_flags;
1493
1494 clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock);
1495 smp_mb__after_atomic();
1496 wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET);
1497}
1498
Andy Adamsond23d61c2012-05-23 05:02:37 -04001499/*
Benny Halevye5e94012010-10-20 00:18:01 -04001500 * Layout segment is retreived from the server if not cached.
1501 * The appropriate layout segment is referenced and returned to the caller.
1502 */
Andy Adamson7c24d942011-06-13 18:22:38 -04001503struct pnfs_layout_segment *
Benny Halevye5e94012010-10-20 00:18:01 -04001504pnfs_update_layout(struct inode *ino,
1505 struct nfs_open_context *ctx,
Benny Halevyfb3296e2011-05-22 19:47:26 +03001506 loff_t pos,
1507 u64 count,
Trond Myklebusta75b9df2011-05-11 18:00:51 -04001508 enum pnfs_iomode iomode,
1509 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -04001510{
Benny Halevyfb3296e2011-05-22 19:47:26 +03001511 struct pnfs_layout_range arg = {
1512 .iomode = iomode,
1513 .offset = pos,
1514 .length = count,
1515 };
Benny Halevy707ed5f2011-05-22 19:47:46 +03001516 unsigned pg_offset;
Weston Andros Adamson6382a442011-06-01 16:44:44 -04001517 struct nfs_server *server = NFS_SERVER(ino);
1518 struct nfs_client *clp = server->nfs_client;
Benny Halevye5e94012010-10-20 00:18:01 -04001519 struct pnfs_layout_hdr *lo;
1520 struct pnfs_layout_segment *lseg = NULL;
Weston Andros Adamson30005122013-02-28 20:30:10 -05001521 bool first;
Benny Halevye5e94012010-10-20 00:18:01 -04001522
1523 if (!pnfs_enabled_sb(NFS_SERVER(ino)))
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001524 goto out;
Andy Adamsond23d61c2012-05-23 05:02:37 -04001525
Trond Myklebust4ae935602015-08-31 01:25:11 -07001526 if (iomode == IOMODE_READ && i_size_read(ino) == 0)
1527 goto out;
1528
Andy Adamsond23d61c2012-05-23 05:02:37 -04001529 if (pnfs_within_mdsthreshold(ctx, ino, iomode))
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001530 goto out;
Andy Adamsond23d61c2012-05-23 05:02:37 -04001531
Peng Tao9bf87482014-08-22 17:37:41 +08001532lookup_again:
1533 first = false;
Benny Halevye5e94012010-10-20 00:18:01 -04001534 spin_lock(&ino->i_lock);
Peng Tao9fa40752011-07-30 20:52:32 -04001535 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001536 if (lo == NULL) {
1537 spin_unlock(&ino->i_lock);
1538 goto out;
1539 }
Benny Halevye5e94012010-10-20 00:18:01 -04001540
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001541 /* Do we even need to bother with this? */
Trond Myklebusta59c30a2012-03-01 11:17:47 -05001542 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001543 dprintk("%s matches recall, use MDS\n", __func__);
Benny Halevye5e94012010-10-20 00:18:01 -04001544 goto out_unlock;
1545 }
1546
1547 /* if LAYOUTGET already failed once we don't try again */
Peng Taoaa8a45e2014-12-01 08:22:23 +08001548 if (pnfs_layout_io_test_failed(lo, iomode) &&
1549 !pnfs_should_retry_layoutget(lo))
Benny Halevye5e94012010-10-20 00:18:01 -04001550 goto out_unlock;
1551
Peng Tao9bf87482014-08-22 17:37:41 +08001552 first = list_empty(&lo->plh_segs);
1553 if (first) {
1554 /* The first layoutget for the file. Need to serialize per
1555 * RFC 5661 Errata 3208.
1556 */
1557 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET,
1558 &lo->plh_flags)) {
1559 spin_unlock(&ino->i_lock);
1560 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_FIRST_LAYOUTGET,
1561 TASK_UNINTERRUPTIBLE);
1562 pnfs_put_layout_hdr(lo);
1563 goto lookup_again;
1564 }
1565 } else {
1566 /* Check to see if the layout for the given range
1567 * already exists
1568 */
1569 lseg = pnfs_find_lseg(lo, &arg);
1570 if (lseg)
1571 goto out_unlock;
1572 }
Andy Adamson568e8c42011-03-01 01:34:22 +00001573
Peng Taoaa8a45e2014-12-01 08:22:23 +08001574 /*
1575 * Because we free lsegs before sending LAYOUTRETURN, we need to wait
1576 * for LAYOUTRETURN even if first is true.
1577 */
Trond Myklebust8f70f532015-08-04 16:09:44 -04001578 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
Peng Taoaa8a45e2014-12-01 08:22:23 +08001579 spin_unlock(&ino->i_lock);
1580 dprintk("%s wait for layoutreturn\n", __func__);
1581 if (pnfs_prepare_to_retry_layoutget(lo)) {
Tom Haynesd67ae822014-12-11 17:02:04 -05001582 if (first)
1583 pnfs_clear_first_layoutget(lo);
Peng Taoaa8a45e2014-12-01 08:22:23 +08001584 pnfs_put_layout_hdr(lo);
1585 dprintk("%s retrying\n", __func__);
1586 goto lookup_again;
1587 }
1588 goto out_put_layout_hdr;
1589 }
1590
Trond Myklebuste1c06f82015-08-04 16:40:08 -04001591 if (pnfs_layoutgets_blocked(lo))
Fred Isamancf7d63f2011-01-06 11:36:25 +00001592 goto out_unlock;
1593 atomic_inc(&lo->plh_outstanding);
Fred Isamanf49f9ba2011-02-03 18:28:52 +00001594 spin_unlock(&ino->i_lock);
Weston Andros Adamson30005122013-02-28 20:30:10 -05001595
Peng Taoabb9a002014-08-22 17:37:40 +08001596 if (list_empty(&lo->plh_layouts)) {
Fred Isaman2130ff62011-01-06 11:36:26 +00001597 /* The lo must be on the clp list if there is any
1598 * chance of a CB_LAYOUTRECALL(FILE) coming in.
1599 */
1600 spin_lock(&clp->cl_lock);
Peng Taoabb9a002014-08-22 17:37:40 +08001601 if (list_empty(&lo->plh_layouts))
1602 list_add_tail(&lo->plh_layouts, &server->layouts);
Fred Isaman2130ff62011-01-06 11:36:26 +00001603 spin_unlock(&clp->cl_lock);
1604 }
Benny Halevye5e94012010-10-20 00:18:01 -04001605
Benny Halevy707ed5f2011-05-22 19:47:46 +03001606 pg_offset = arg.offset & ~PAGE_CACHE_MASK;
1607 if (pg_offset) {
1608 arg.offset -= pg_offset;
1609 arg.length += pg_offset;
1610 }
Andy Adamson7c24d942011-06-13 18:22:38 -04001611 if (arg.length != NFS4_MAX_UINT64)
1612 arg.length = PAGE_CACHE_ALIGN(arg.length);
Benny Halevy707ed5f2011-05-22 19:47:46 +03001613
Benny Halevyfb3296e2011-05-22 19:47:26 +03001614 lseg = send_layoutget(lo, ctx, &arg, gfp_flags);
Peng Taoc8290132014-12-01 08:22:18 +08001615 pnfs_clear_retry_layoutget(lo);
Fred Isamancf7d63f2011-01-06 11:36:25 +00001616 atomic_dec(&lo->plh_outstanding);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001617out_put_layout_hdr:
Tom Haynesd67ae822014-12-11 17:02:04 -05001618 if (first)
1619 pnfs_clear_first_layoutget(lo);
Trond Myklebust70c3bd22012-09-18 20:51:13 -04001620 pnfs_put_layout_hdr(lo);
Benny Halevye5e94012010-10-20 00:18:01 -04001621out:
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001622 dprintk("%s: inode %s/%llu pNFS layout segment %s for "
1623 "(%s, offset: %llu, length: %llu)\n",
1624 __func__, ino->i_sb->s_id,
1625 (unsigned long long)NFS_FILEID(ino),
1626 lseg == NULL ? "not found" : "found",
1627 iomode==IOMODE_RW ? "read/write" : "read-only",
1628 (unsigned long long)pos,
1629 (unsigned long long)count);
Benny Halevye5e94012010-10-20 00:18:01 -04001630 return lseg;
1631out_unlock:
1632 spin_unlock(&ino->i_lock);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001633 goto out_put_layout_hdr;
Benny Halevye5e94012010-10-20 00:18:01 -04001634}
Andy Adamson7c24d942011-06-13 18:22:38 -04001635EXPORT_SYMBOL_GPL(pnfs_update_layout);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001636
Trond Myklebust540d9862015-08-25 11:16:13 -04001637static bool
1638pnfs_sanity_check_layout_range(struct pnfs_layout_range *range)
1639{
1640 switch (range->iomode) {
1641 case IOMODE_READ:
1642 case IOMODE_RW:
1643 break;
1644 default:
1645 return false;
1646 }
1647 if (range->offset == NFS4_MAX_UINT64)
1648 return false;
1649 if (range->length == 0)
1650 return false;
1651 if (range->length != NFS4_MAX_UINT64 &&
1652 range->length > NFS4_MAX_UINT64 - range->offset)
1653 return false;
1654 return true;
1655}
1656
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -04001657struct pnfs_layout_segment *
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001658pnfs_layout_process(struct nfs4_layoutget *lgp)
1659{
1660 struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout;
1661 struct nfs4_layoutget_res *res = &lgp->res;
1662 struct pnfs_layout_segment *lseg;
Fred Isamanb7edfaa2011-01-06 11:36:21 +00001663 struct inode *ino = lo->plh_inode;
Trond Myklebust78096cc2014-02-12 10:02:27 -05001664 LIST_HEAD(free_me);
Trond Myklebust540d9862015-08-25 11:16:13 -04001665 int status = -EINVAL;
1666
1667 if (!pnfs_sanity_check_layout_range(&res->range))
1668 goto out;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001669
1670 /* Inject layout blob into I/O device driver */
Trond Myklebusta75b9df2011-05-11 18:00:51 -04001671 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001672 if (!lseg || IS_ERR(lseg)) {
1673 if (!lseg)
1674 status = -ENOMEM;
1675 else
1676 status = PTR_ERR(lseg);
1677 dprintk("%s: Could not allocate layout: error %d\n",
1678 __func__, status);
1679 goto out;
1680 }
1681
Christoph Hellwig1013df612014-08-21 11:09:18 -05001682 init_lseg(lo, lseg);
1683 lseg->pls_range = res->range;
1684
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001685 spin_lock(&ino->i_lock);
Trond Myklebuste1c06f82015-08-04 16:40:08 -04001686 if (pnfs_layoutgets_blocked(lo)) {
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001687 dprintk("%s forget reply due to state\n", __func__);
1688 goto out_forget_reply;
1689 }
Trond Myklebust038d6492012-10-02 16:38:41 -07001690
Christoph Hellwig362f7472014-08-21 11:09:20 -05001691 if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) {
1692 /* existing state ID, make sure the sequence number matches. */
1693 if (pnfs_layout_stateid_blocked(lo, &res->stateid)) {
1694 dprintk("%s forget reply due to sequence\n", __func__);
Jeff Layton4f2e9dc2015-11-25 13:43:14 -05001695 status = -EAGAIN;
Christoph Hellwig362f7472014-08-21 11:09:20 -05001696 goto out_forget_reply;
1697 }
1698 pnfs_set_layout_stateid(lo, &res->stateid, false);
1699 } else {
1700 /*
1701 * We got an entirely new state ID. Mark all segments for the
1702 * inode invalid, and don't bother validating the stateid
1703 * sequence number.
1704 */
1705 pnfs_mark_matching_lsegs_invalid(lo, &free_me, NULL);
1706
1707 nfs4_stateid_copy(&lo->plh_stateid, &res->stateid);
1708 lo->plh_barrier = be32_to_cpu(res->stateid.seqid);
1709 }
Trond Myklebust038d6492012-10-02 16:38:41 -07001710
Christoph Hellwig47abade2014-08-21 11:09:22 -05001711 clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
1712
Trond Myklebust9369a432012-09-18 20:57:08 -04001713 pnfs_get_lseg(lseg);
Trond Myklebust03772d22015-08-25 08:54:17 -04001714 pnfs_layout_insert_lseg(lo, lseg, &free_me);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001715
Peng Tao39761432015-08-21 12:49:44 +08001716 if (res->return_on_close)
Fred Isamanf7e89172011-01-06 11:36:32 +00001717 set_bit(NFS_LSEG_ROC, &lseg->pls_flags);
Fred Isamanf7e89172011-01-06 11:36:32 +00001718
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001719 spin_unlock(&ino->i_lock);
Trond Myklebust78096cc2014-02-12 10:02:27 -05001720 pnfs_free_lseg_list(&free_me);
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -04001721 return lseg;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001722out:
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -04001723 return ERR_PTR(status);
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001724
1725out_forget_reply:
1726 spin_unlock(&ino->i_lock);
1727 lseg->pls_layout = lo;
1728 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
1729 goto out;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001730}
1731
Peng Tao016256d2014-09-06 00:53:23 +08001732static void
1733pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
1734 struct list_head *tmp_list,
1735 struct pnfs_layout_range *return_range)
1736{
1737 struct pnfs_layout_segment *lseg, *next;
1738
1739 dprintk("%s:Begin lo %p\n", __func__, lo);
1740
1741 if (list_empty(&lo->plh_segs))
1742 return;
1743
1744 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
1745 if (should_free_lseg(&lseg->pls_range, return_range)) {
1746 dprintk("%s: marking lseg %p iomode %d "
1747 "offset %llu length %llu\n", __func__,
1748 lseg, lseg->pls_range.iomode,
1749 lseg->pls_range.offset,
1750 lseg->pls_range.length);
1751 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
1752 mark_lseg_invalid(lseg, tmp_list);
Trond Myklebust0bdb8fa2015-08-27 19:17:33 -04001753 set_bit(NFS_LAYOUT_RETURN_BEFORE_CLOSE,
1754 &lo->plh_flags);
Peng Tao016256d2014-09-06 00:53:23 +08001755 }
1756}
1757
1758void pnfs_error_mark_layout_for_return(struct inode *inode,
1759 struct pnfs_layout_segment *lseg)
1760{
1761 struct pnfs_layout_hdr *lo = NFS_I(inode)->layout;
1762 int iomode = pnfs_iomode_to_fail_bit(lseg->pls_range.iomode);
1763 struct pnfs_layout_range range = {
1764 .iomode = lseg->pls_range.iomode,
1765 .offset = 0,
1766 .length = NFS4_MAX_UINT64,
1767 };
1768 LIST_HEAD(free_me);
1769
1770 spin_lock(&inode->i_lock);
1771 /* set failure bit so that pnfs path will be retried later */
1772 pnfs_layout_set_fail_bit(lo, iomode);
Peng Tao016256d2014-09-06 00:53:23 +08001773 if (lo->plh_return_iomode == 0)
1774 lo->plh_return_iomode = range.iomode;
1775 else if (lo->plh_return_iomode != range.iomode)
1776 lo->plh_return_iomode = IOMODE_ANY;
1777 /*
1778 * mark all matching lsegs so that we are sure to have no live
1779 * segments at hand when sending layoutreturn. See pnfs_put_lseg()
1780 * for how it works.
1781 */
1782 pnfs_mark_matching_lsegs_return(lo, &free_me, &range);
1783 spin_unlock(&inode->i_lock);
1784 pnfs_free_lseg_list(&free_me);
1785}
1786EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return);
1787
Trond Myklebustd8007d42011-06-10 13:30:23 -04001788void
1789pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
1790{
Peng Tao1fd937b2012-09-25 14:55:57 +08001791 u64 rd_size = req->wb_bytes;
1792
Peng Taocb5d04b2015-01-24 22:14:52 +08001793 if (pgio->pg_lseg == NULL) {
1794 if (pgio->pg_dreq == NULL)
1795 rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
1796 else
1797 rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);
Trond Myklebustd8007d42011-06-10 13:30:23 -04001798
Peng Taocb5d04b2015-01-24 22:14:52 +08001799 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
1800 req->wb_context,
1801 req_offset(req),
1802 rd_size,
1803 IOMODE_READ,
1804 GFP_KERNEL);
1805 }
Trond Myklebuste885de12011-06-10 13:30:23 -04001806 /* If no lseg, fall back to read through mds */
1807 if (pgio->pg_lseg == NULL)
Trond Myklebust1f945352011-07-13 15:59:57 -04001808 nfs_pageio_reset_read_mds(pgio);
Trond Myklebuste885de12011-06-10 13:30:23 -04001809
Trond Myklebustd8007d42011-06-10 13:30:23 -04001810}
1811EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read);
1812
1813void
Peng Tao62965562012-09-25 14:55:57 +08001814pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
1815 struct nfs_page *req, u64 wb_size)
Trond Myklebustd8007d42011-06-10 13:30:23 -04001816{
Peng Taocb5d04b2015-01-24 22:14:52 +08001817 if (pgio->pg_lseg == NULL)
1818 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
1819 req->wb_context,
1820 req_offset(req),
1821 wb_size,
1822 IOMODE_RW,
1823 GFP_NOFS);
Trond Myklebuste885de12011-06-10 13:30:23 -04001824 /* If no lseg, fall back to write through mds */
1825 if (pgio->pg_lseg == NULL)
Trond Myklebust1f945352011-07-13 15:59:57 -04001826 nfs_pageio_reset_write_mds(pgio);
Trond Myklebustd8007d42011-06-10 13:30:23 -04001827}
1828EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);
1829
Weston Andros Adamson180bb5e2014-09-10 15:48:01 -04001830void
1831pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc)
1832{
1833 if (desc->pg_lseg) {
1834 pnfs_put_lseg(desc->pg_lseg);
1835 desc->pg_lseg = NULL;
1836 }
1837}
1838EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup);
1839
Weston Andros Adamsonb4fdac12014-05-15 11:56:43 -04001840/*
1841 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
1842 * of bytes (maximum @req->wb_bytes) that can be coalesced.
1843 */
1844size_t
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04001845pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio,
1846 struct nfs_page *prev, struct nfs_page *req)
Fred Isamanbae724e2011-03-01 01:34:15 +00001847{
Weston Andros Adamson0f9c4292014-05-15 11:56:51 -04001848 unsigned int size;
Weston Andros Adamsonc5e20cb2014-06-09 17:47:26 -04001849 u64 seg_end, req_start, seg_left;
Weston Andros Adamson0f9c4292014-05-15 11:56:51 -04001850
1851 size = nfs_generic_pg_test(pgio, prev, req);
Weston Andros Adamson0f9c4292014-05-15 11:56:51 -04001852 if (!size)
1853 return 0;
Benny Halevy89a58e32011-05-25 20:54:40 +03001854
Trond Myklebust19982ba2011-06-10 13:30:23 -04001855 /*
Weston Andros Adamsonc5e20cb2014-06-09 17:47:26 -04001856 * 'size' contains the number of bytes left in the current page (up
1857 * to the original size asked for in @req->wb_bytes).
1858 *
1859 * Calculate how many bytes are left in the layout segment
1860 * and if there are less bytes than 'size', return that instead.
Trond Myklebust19982ba2011-06-10 13:30:23 -04001861 *
1862 * Please also note that 'end_offset' is actually the offset of the
1863 * first byte that lies outside the pnfs_layout_range. FIXME?
1864 *
1865 */
Weston Andros Adamson19b54842014-05-15 11:56:55 -04001866 if (pgio->pg_lseg) {
Weston Andros Adamsonc5e20cb2014-06-09 17:47:26 -04001867 seg_end = end_offset(pgio->pg_lseg->pls_range.offset,
1868 pgio->pg_lseg->pls_range.length);
1869 req_start = req_offset(req);
Weston Andros Adamson7c137892015-01-30 11:01:02 -05001870 WARN_ON_ONCE(req_start >= seg_end);
Weston Andros Adamsonc5e20cb2014-06-09 17:47:26 -04001871 /* start of request is past the last byte of this segment */
Weston Andros Adamson7c137892015-01-30 11:01:02 -05001872 if (req_start >= seg_end) {
1873 /* reference the new lseg */
1874 if (pgio->pg_ops->pg_cleanup)
1875 pgio->pg_ops->pg_cleanup(pgio);
1876 if (pgio->pg_ops->pg_init)
1877 pgio->pg_ops->pg_init(pgio, req);
Weston Andros Adamson19b54842014-05-15 11:56:55 -04001878 return 0;
Weston Andros Adamson7c137892015-01-30 11:01:02 -05001879 }
Weston Andros Adamsonc5e20cb2014-06-09 17:47:26 -04001880
1881 /* adjust 'size' iff there are fewer bytes left in the
1882 * segment than what nfs_generic_pg_test returned */
1883 seg_left = seg_end - req_start;
1884 if (seg_left < size)
1885 size = (unsigned int)seg_left;
Weston Andros Adamson19b54842014-05-15 11:56:55 -04001886 }
Weston Andros Adamson0f9c4292014-05-15 11:56:51 -04001887
Weston Andros Adamson19b54842014-05-15 11:56:55 -04001888 return size;
Fred Isamanbae724e2011-03-01 01:34:15 +00001889}
Benny Halevy89a58e32011-05-25 20:54:40 +03001890EXPORT_SYMBOL_GPL(pnfs_generic_pg_test);
Fred Isamanbae724e2011-03-01 01:34:15 +00001891
Weston Andros Adamson53113ad2014-06-09 11:48:38 -04001892int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr)
Trond Myklebuste2fecb22012-01-06 08:57:46 -05001893{
1894 struct nfs_pageio_descriptor pgio;
Trond Myklebuste2fecb22012-01-06 08:57:46 -05001895
1896 /* Resend all requests through the MDS */
Weston Andros Adamson53113ad2014-06-09 11:48:38 -04001897 nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true,
1898 hdr->completion_ops);
Trond Myklebustc7070112015-06-17 19:56:22 -04001899 set_bit(NFS_CONTEXT_RESEND_WRITES, &hdr->args.context->flags);
Weston Andros Adamson53113ad2014-06-09 11:48:38 -04001900 return nfs_pageio_resend(&pgio, hdr);
Trond Myklebuste2fecb22012-01-06 08:57:46 -05001901}
Andy Adamsone7dd79af2012-04-27 17:53:46 -04001902EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds);
Trond Myklebuste2fecb22012-01-06 08:57:46 -05001903
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04001904static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr)
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001905{
Fred Isamancd841602012-04-20 14:47:44 -04001906
1907 dprintk("pnfs write error = %d\n", hdr->pnfs_error);
1908 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001909 PNFS_LAYOUTRET_ON_ERROR) {
Fred Isamancd841602012-04-20 14:47:44 -04001910 pnfs_return_layout(hdr->inode);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001911 }
Fred Isaman6c75dc02012-04-20 14:47:47 -04001912 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
Weston Andros Adamson53113ad2014-06-09 11:48:38 -04001913 hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04001914}
1915
Benny Halevyd20581a2011-05-22 19:52:03 +03001916/*
1917 * Called by non rpc-based layout drivers
1918 */
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04001919void pnfs_ld_write_done(struct nfs_pgio_header *hdr)
Fred Isaman94ad1c82011-03-01 01:34:14 +00001920{
Kinglong Meef8417b482015-10-16 17:23:29 +08001921 if (likely(!hdr->pnfs_error)) {
Trond Myklebust67af7612015-03-25 20:40:38 -04001922 pnfs_set_layoutcommit(hdr->inode, hdr->lseg,
1923 hdr->mds_offset + hdr->res.count);
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04001924 hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
Kinglong Meef8417b482015-10-16 17:23:29 +08001925 }
1926 trace_nfs4_pnfs_write(hdr, hdr->pnfs_error);
1927 if (unlikely(hdr->pnfs_error))
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04001928 pnfs_ld_handle_write_error(hdr);
1929 hdr->mds_ops->rpc_release(hdr);
Fred Isaman44b83792011-03-03 15:13:44 +00001930}
Benny Halevyd20581a2011-05-22 19:52:03 +03001931EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
Fred Isaman44b83792011-03-03 15:13:44 +00001932
Trond Myklebustdce81292011-07-13 15:59:19 -04001933static void
1934pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04001935 struct nfs_pgio_header *hdr)
Trond Myklebustdce81292011-07-13 15:59:19 -04001936{
Peng Tao48d635f2014-11-10 08:35:35 +08001937 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04001938
Fred Isaman6c75dc02012-04-20 14:47:47 -04001939 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04001940 list_splice_tail_init(&hdr->pages, &mirror->pg_list);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001941 nfs_pageio_reset_write_mds(desc);
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04001942 mirror->pg_recoalesce = 1;
Fred Isaman6c75dc02012-04-20 14:47:47 -04001943 }
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04001944 nfs_pgio_data_destroy(hdr);
Trond Myklebust1ca018d2015-06-17 19:41:51 -04001945 hdr->release(hdr);
Trond Myklebustdce81292011-07-13 15:59:19 -04001946}
1947
1948static enum pnfs_try_status
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04001949pnfs_try_to_write_data(struct nfs_pgio_header *hdr,
Trond Myklebustdce81292011-07-13 15:59:19 -04001950 const struct rpc_call_ops *call_ops,
1951 struct pnfs_layout_segment *lseg,
1952 int how)
Andy Adamson0382b742011-03-03 15:13:45 +00001953{
Fred Isamancd841602012-04-20 14:47:44 -04001954 struct inode *inode = hdr->inode;
Andy Adamson0382b742011-03-03 15:13:45 +00001955 enum pnfs_try_status trypnfs;
1956 struct nfs_server *nfss = NFS_SERVER(inode);
1957
Fred Isamancd841602012-04-20 14:47:44 -04001958 hdr->mds_ops = call_ops;
Andy Adamson0382b742011-03-03 15:13:45 +00001959
1960 dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04001961 inode->i_ino, hdr->args.count, hdr->args.offset, how);
1962 trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001963 if (trypnfs != PNFS_NOT_ATTEMPTED)
Andy Adamson0382b742011-03-03 15:13:45 +00001964 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE);
Andy Adamson0382b742011-03-03 15:13:45 +00001965 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
1966 return trypnfs;
1967}
1968
Trond Myklebustdce81292011-07-13 15:59:19 -04001969static void
Weston Andros Adamson7f714722014-05-15 11:56:53 -04001970pnfs_do_write(struct nfs_pageio_descriptor *desc,
1971 struct nfs_pgio_header *hdr, int how)
Trond Myklebustdce81292011-07-13 15:59:19 -04001972{
Trond Myklebustdce81292011-07-13 15:59:19 -04001973 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
1974 struct pnfs_layout_segment *lseg = desc->pg_lseg;
Weston Andros Adamson7f714722014-05-15 11:56:53 -04001975 enum pnfs_try_status trypnfs;
Trond Myklebustdce81292011-07-13 15:59:19 -04001976
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04001977 trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how);
Weston Andros Adamson7f714722014-05-15 11:56:53 -04001978 if (trypnfs == PNFS_NOT_ATTEMPTED)
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04001979 pnfs_write_through_mds(desc, hdr);
Trond Myklebustdce81292011-07-13 15:59:19 -04001980}
1981
Fred Isaman6c75dc02012-04-20 14:47:47 -04001982static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
1983{
Trond Myklebust9369a432012-09-18 20:57:08 -04001984 pnfs_put_lseg(hdr->lseg);
Weston Andros Adamson1e7f3a42014-06-09 11:48:33 -04001985 nfs_pgio_header_free(hdr);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001986}
1987
Trond Myklebustdce81292011-07-13 15:59:19 -04001988int
1989pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
1990{
Peng Tao48d635f2014-11-10 08:35:35 +08001991 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04001992
Fred Isaman6c75dc02012-04-20 14:47:47 -04001993 struct nfs_pgio_header *hdr;
Trond Myklebustdce81292011-07-13 15:59:19 -04001994 int ret;
1995
Weston Andros Adamson1e7f3a42014-06-09 11:48:33 -04001996 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
1997 if (!hdr) {
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04001998 desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
Fred Isaman6c75dc02012-04-20 14:47:47 -04001999 return -ENOMEM;
2000 }
Fred Isaman6c75dc02012-04-20 14:47:47 -04002001 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
Weston Andros Adamson180bb5e2014-09-10 15:48:01 -04002002
Trond Myklebust9369a432012-09-18 20:57:08 -04002003 hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
Anna Schumakeref2c4882014-05-06 09:12:36 -04002004 ret = nfs_generic_pgio(desc, hdr);
Weston Andros Adamson180bb5e2014-09-10 15:48:01 -04002005 if (!ret)
Weston Andros Adamson7f714722014-05-15 11:56:53 -04002006 pnfs_do_write(desc, hdr, desc->pg_ioflags);
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04002007
Fred Isaman6c75dc02012-04-20 14:47:47 -04002008 return ret;
Trond Myklebustdce81292011-07-13 15:59:19 -04002009}
2010EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);
2011
Weston Andros Adamson53113ad2014-06-09 11:48:38 -04002012int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr)
Trond Myklebust62e4a762011-11-10 14:30:37 -05002013{
2014 struct nfs_pageio_descriptor pgio;
2015
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002016 /* Resend all requests through the MDS */
Weston Andros Adamson53113ad2014-06-09 11:48:38 -04002017 nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops);
2018 return nfs_pageio_resend(&pgio, hdr);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002019}
Andy Adamsone7dd79af2012-04-27 17:53:46 -04002020EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002021
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002022static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr)
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002023{
Fred Isamancd841602012-04-20 14:47:44 -04002024 dprintk("pnfs read error = %d\n", hdr->pnfs_error);
2025 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002026 PNFS_LAYOUTRET_ON_ERROR) {
Fred Isamancd841602012-04-20 14:47:44 -04002027 pnfs_return_layout(hdr->inode);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002028 }
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002029 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
Weston Andros Adamson53113ad2014-06-09 11:48:38 -04002030 hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr);
Trond Myklebust62e4a762011-11-10 14:30:37 -05002031}
2032
Andy Adamsonb1f69b72010-10-20 00:18:03 -04002033/*
Benny Halevyd20581a2011-05-22 19:52:03 +03002034 * Called by non rpc-based layout drivers
2035 */
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002036void pnfs_ld_read_done(struct nfs_pgio_header *hdr)
Benny Halevyd20581a2011-05-22 19:52:03 +03002037{
Fred Isamancd841602012-04-20 14:47:44 -04002038 if (likely(!hdr->pnfs_error)) {
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002039 __nfs4_read_done_cb(hdr);
2040 hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
Kinglong Meef8417b482015-10-16 17:23:29 +08002041 }
2042 trace_nfs4_pnfs_read(hdr, hdr->pnfs_error);
2043 if (unlikely(hdr->pnfs_error))
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002044 pnfs_ld_handle_read_error(hdr);
2045 hdr->mds_ops->rpc_release(hdr);
Benny Halevyd20581a2011-05-22 19:52:03 +03002046}
2047EXPORT_SYMBOL_GPL(pnfs_ld_read_done);
2048
Trond Myklebust493292d2011-07-13 15:58:28 -04002049static void
2050pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002051 struct nfs_pgio_header *hdr)
Trond Myklebust493292d2011-07-13 15:58:28 -04002052{
Peng Tao48d635f2014-11-10 08:35:35 +08002053 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04002054
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002055 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04002056 list_splice_tail_init(&hdr->pages, &mirror->pg_list);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002057 nfs_pageio_reset_read_mds(desc);
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04002058 mirror->pg_recoalesce = 1;
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002059 }
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002060 nfs_pgio_data_destroy(hdr);
Trond Myklebust1ca018d2015-06-17 19:41:51 -04002061 hdr->release(hdr);
Trond Myklebust493292d2011-07-13 15:58:28 -04002062}
2063
Benny Halevyd20581a2011-05-22 19:52:03 +03002064/*
Andy Adamson64419a92011-03-01 01:34:16 +00002065 * Call the appropriate parallel I/O subsystem read function.
2066 */
Trond Myklebust493292d2011-07-13 15:58:28 -04002067static enum pnfs_try_status
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002068pnfs_try_to_read_data(struct nfs_pgio_header *hdr,
Trond Myklebust493292d2011-07-13 15:58:28 -04002069 const struct rpc_call_ops *call_ops,
2070 struct pnfs_layout_segment *lseg)
Andy Adamson64419a92011-03-01 01:34:16 +00002071{
Fred Isamancd841602012-04-20 14:47:44 -04002072 struct inode *inode = hdr->inode;
Andy Adamson64419a92011-03-01 01:34:16 +00002073 struct nfs_server *nfss = NFS_SERVER(inode);
2074 enum pnfs_try_status trypnfs;
2075
Fred Isamancd841602012-04-20 14:47:44 -04002076 hdr->mds_ops = call_ops;
Andy Adamson64419a92011-03-01 01:34:16 +00002077
2078 dprintk("%s: Reading ino:%lu %u@%llu\n",
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002079 __func__, inode->i_ino, hdr->args.count, hdr->args.offset);
Andy Adamson64419a92011-03-01 01:34:16 +00002080
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002081 trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002082 if (trypnfs != PNFS_NOT_ATTEMPTED)
Andy Adamson64419a92011-03-01 01:34:16 +00002083 nfs_inc_stats(inode, NFSIOS_PNFS_READ);
Andy Adamson64419a92011-03-01 01:34:16 +00002084 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
2085 return trypnfs;
2086}
Andy Adamson863a3c62011-03-23 13:27:54 +00002087
Peng Taoceb11e12014-11-10 08:35:38 +08002088/* Resend all requests through pnfs. */
2089int pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr)
2090{
2091 struct nfs_pageio_descriptor pgio;
2092
2093 nfs_pageio_init_read(&pgio, hdr->inode, false, hdr->completion_ops);
2094 return nfs_pageio_resend(&pgio, hdr);
2095}
2096EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs);
2097
Trond Myklebust493292d2011-07-13 15:58:28 -04002098static void
Weston Andros Adamson7f714722014-05-15 11:56:53 -04002099pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr)
Trond Myklebust493292d2011-07-13 15:58:28 -04002100{
Trond Myklebust493292d2011-07-13 15:58:28 -04002101 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
2102 struct pnfs_layout_segment *lseg = desc->pg_lseg;
Weston Andros Adamson7f714722014-05-15 11:56:53 -04002103 enum pnfs_try_status trypnfs;
Peng Taoceb11e12014-11-10 08:35:38 +08002104 int err = 0;
Trond Myklebust493292d2011-07-13 15:58:28 -04002105
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002106 trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg);
Peng Taoceb11e12014-11-10 08:35:38 +08002107 if (trypnfs == PNFS_TRY_AGAIN)
2108 err = pnfs_read_resend_pnfs(hdr);
2109 if (trypnfs == PNFS_NOT_ATTEMPTED || err)
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002110 pnfs_read_through_mds(desc, hdr);
Trond Myklebust493292d2011-07-13 15:58:28 -04002111}
2112
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002113static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
2114{
Trond Myklebust9369a432012-09-18 20:57:08 -04002115 pnfs_put_lseg(hdr->lseg);
Weston Andros Adamson1e7f3a42014-06-09 11:48:33 -04002116 nfs_pgio_header_free(hdr);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002117}
2118
Trond Myklebust493292d2011-07-13 15:58:28 -04002119int
2120pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
2121{
Peng Tao48d635f2014-11-10 08:35:35 +08002122 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04002123
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002124 struct nfs_pgio_header *hdr;
Trond Myklebust493292d2011-07-13 15:58:28 -04002125 int ret;
2126
Weston Andros Adamson1e7f3a42014-06-09 11:48:33 -04002127 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
2128 if (!hdr) {
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04002129 desc->pg_completion_ops->error_cleanup(&mirror->pg_list);
Weston Andros Adamson180bb5e2014-09-10 15:48:01 -04002130 return -ENOMEM;
Trond Myklebust493292d2011-07-13 15:58:28 -04002131 }
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002132 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
Trond Myklebust9369a432012-09-18 20:57:08 -04002133 hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
Anna Schumakeref2c4882014-05-06 09:12:36 -04002134 ret = nfs_generic_pgio(desc, hdr);
Weston Andros Adamson180bb5e2014-09-10 15:48:01 -04002135 if (!ret)
Weston Andros Adamson7f714722014-05-15 11:56:53 -04002136 pnfs_do_read(desc, hdr);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002137 return ret;
Trond Myklebust493292d2011-07-13 15:58:28 -04002138}
2139EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages);
2140
Trond Myklebust71244d92014-01-13 13:34:36 -05002141static void pnfs_clear_layoutcommitting(struct inode *inode)
2142{
2143 unsigned long *bitlock = &NFS_I(inode)->flags;
2144
2145 clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01002146 smp_mb__after_atomic();
Trond Myklebust71244d92014-01-13 13:34:36 -05002147 wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING);
2148}
2149
Andy Adamson863a3c62011-03-23 13:27:54 +00002150/*
Peng Taoa9bae562011-07-30 20:52:33 -04002151 * There can be multiple RW segments.
Andy Adamson863a3c62011-03-23 13:27:54 +00002152 */
Peng Taoa9bae562011-07-30 20:52:33 -04002153static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
Andy Adamson863a3c62011-03-23 13:27:54 +00002154{
Peng Taoa9bae562011-07-30 20:52:33 -04002155 struct pnfs_layout_segment *lseg;
Andy Adamson863a3c62011-03-23 13:27:54 +00002156
Peng Taoa9bae562011-07-30 20:52:33 -04002157 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
2158 if (lseg->pls_range.iomode == IOMODE_RW &&
Trond Myklebusta073dbf2013-03-20 12:34:32 -04002159 test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
Peng Taoa9bae562011-07-30 20:52:33 -04002160 list_add(&lseg->pls_lc_list, listp);
2161 }
Andy Adamson863a3c62011-03-23 13:27:54 +00002162}
2163
Trond Myklebusta073dbf2013-03-20 12:34:32 -04002164static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp)
2165{
2166 struct pnfs_layout_segment *lseg, *tmp;
Trond Myklebusta073dbf2013-03-20 12:34:32 -04002167
2168 /* Matched by references in pnfs_set_layoutcommit */
2169 list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) {
2170 list_del_init(&lseg->pls_lc_list);
2171 pnfs_put_lseg(lseg);
2172 }
2173
Trond Myklebust71244d92014-01-13 13:34:36 -05002174 pnfs_clear_layoutcommitting(inode);
Trond Myklebusta073dbf2013-03-20 12:34:32 -04002175}
2176
Peng Tao1b0ae062011-09-22 21:50:12 -04002177void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
2178{
Trond Myklebustb9e028f2012-09-18 16:41:18 -04002179 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
Peng Tao1b0ae062011-09-22 21:50:12 -04002180}
2181EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);
2182
Andy Adamson863a3c62011-03-23 13:27:54 +00002183void
Trond Myklebust67af7612015-03-25 20:40:38 -04002184pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg,
2185 loff_t end_pos)
Andy Adamson863a3c62011-03-23 13:27:54 +00002186{
Fred Isamancd841602012-04-20 14:47:44 -04002187 struct nfs_inode *nfsi = NFS_I(inode);
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04002188 bool mark_as_dirty = false;
Andy Adamson863a3c62011-03-23 13:27:54 +00002189
Fred Isamancd841602012-04-20 14:47:44 -04002190 spin_lock(&inode->i_lock);
Andy Adamson863a3c62011-03-23 13:27:54 +00002191 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
Trond Myklebust29559b12015-03-25 20:10:20 -04002192 nfsi->layout->plh_lwb = end_pos;
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04002193 mark_as_dirty = true;
Andy Adamson863a3c62011-03-23 13:27:54 +00002194 dprintk("%s: Set layoutcommit for inode %lu ",
Fred Isamancd841602012-04-20 14:47:44 -04002195 __func__, inode->i_ino);
Trond Myklebust29559b12015-03-25 20:10:20 -04002196 } else if (end_pos > nfsi->layout->plh_lwb)
2197 nfsi->layout->plh_lwb = end_pos;
Trond Myklebust67af7612015-03-25 20:40:38 -04002198 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) {
Peng Taoa9bae562011-07-30 20:52:33 -04002199 /* references matched in nfs4_layoutcommit_release */
Trond Myklebust67af7612015-03-25 20:40:38 -04002200 pnfs_get_lseg(lseg);
Peng Taoa9bae562011-07-30 20:52:33 -04002201 }
Fred Isamancd841602012-04-20 14:47:44 -04002202 spin_unlock(&inode->i_lock);
Peng Taoacff58802011-07-30 20:52:31 -04002203 dprintk("%s: lseg %p end_pos %llu\n",
Trond Myklebust67af7612015-03-25 20:40:38 -04002204 __func__, lseg, nfsi->layout->plh_lwb);
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04002205
2206 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one
2207 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
2208 if (mark_as_dirty)
Fred Isamancd841602012-04-20 14:47:44 -04002209 mark_inode_dirty_sync(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00002210}
2211EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);
2212
Andy Adamsondb29c082011-07-30 20:52:38 -04002213void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data)
2214{
2215 struct nfs_server *nfss = NFS_SERVER(data->args.inode);
2216
2217 if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
2218 nfss->pnfs_curr_ld->cleanup_layoutcommit(data);
Trond Myklebusta073dbf2013-03-20 12:34:32 -04002219 pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list);
Andy Adamsondb29c082011-07-30 20:52:38 -04002220}
2221
Andy Adamsonde4b15c2011-03-12 02:58:09 -05002222/*
2223 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
2224 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough
2225 * data to disk to allow the server to recover the data if it crashes.
2226 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag
2227 * is off, and a COMMIT is sent to a data server, or
2228 * if WRITEs to a data server return NFS_DATA_SYNC.
2229 */
Andy Adamson863a3c62011-03-23 13:27:54 +00002230int
Andy Adamsonef311532011-03-12 02:58:10 -05002231pnfs_layoutcommit_inode(struct inode *inode, bool sync)
Andy Adamson863a3c62011-03-23 13:27:54 +00002232{
Christoph Hellwig5f919c92014-08-21 11:09:25 -05002233 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
Andy Adamson863a3c62011-03-23 13:27:54 +00002234 struct nfs4_layoutcommit_data *data;
2235 struct nfs_inode *nfsi = NFS_I(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00002236 loff_t end_pos;
Trond Myklebust71244d92014-01-13 13:34:36 -05002237 int status;
2238
2239 if (!pnfs_layoutcommit_outstanding(inode))
2240 return 0;
Andy Adamson863a3c62011-03-23 13:27:54 +00002241
2242 dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
2243
Trond Myklebust71244d92014-01-13 13:34:36 -05002244 status = -EAGAIN;
2245 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
2246 if (!sync)
2247 goto out;
NeilBrown74316202014-07-07 15:16:04 +10002248 status = wait_on_bit_lock_action(&nfsi->flags,
Trond Myklebust71244d92014-01-13 13:34:36 -05002249 NFS_INO_LAYOUTCOMMITTING,
2250 nfs_wait_bit_killable,
2251 TASK_KILLABLE);
2252 if (status)
2253 goto out;
2254 }
Andy Adamsonde4b15c2011-03-12 02:58:09 -05002255
Trond Myklebust71244d92014-01-13 13:34:36 -05002256 status = -ENOMEM;
Andy Adamson863a3c62011-03-23 13:27:54 +00002257 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */
2258 data = kzalloc(sizeof(*data), GFP_NOFS);
Trond Myklebust71244d92014-01-13 13:34:36 -05002259 if (!data)
2260 goto clear_layoutcommitting;
Andy Adamson863a3c62011-03-23 13:27:54 +00002261
Trond Myklebust71244d92014-01-13 13:34:36 -05002262 status = 0;
2263 spin_lock(&inode->i_lock);
2264 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
2265 goto out_unlock;
Peng Tao92407e72011-10-23 20:21:17 -07002266
Peng Taoa9bae562011-07-30 20:52:33 -04002267 INIT_LIST_HEAD(&data->lseg_list);
Peng Taoa9bae562011-07-30 20:52:33 -04002268 pnfs_list_write_lseg(inode, &data->lseg_list);
Andy Adamson863a3c62011-03-23 13:27:54 +00002269
Peng Taoacff58802011-07-30 20:52:31 -04002270 end_pos = nfsi->layout->plh_lwb;
Andy Adamson863a3c62011-03-23 13:27:54 +00002271
Trond Myklebustf597c532012-03-04 18:13:56 -05002272 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid);
Andy Adamson863a3c62011-03-23 13:27:54 +00002273 spin_unlock(&inode->i_lock);
2274
2275 data->args.inode = inode;
Peng Tao9fa40752011-07-30 20:52:32 -04002276 data->cred = get_rpccred(nfsi->layout->plh_lc_cred);
Andy Adamson863a3c62011-03-23 13:27:54 +00002277 nfs_fattr_init(&data->fattr);
2278 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
2279 data->res.fattr = &data->fattr;
2280 data->args.lastbytewritten = end_pos - 1;
2281 data->res.server = NFS_SERVER(inode);
2282
Christoph Hellwig5f919c92014-08-21 11:09:25 -05002283 if (ld->prepare_layoutcommit) {
2284 status = ld->prepare_layoutcommit(&data->args);
2285 if (status) {
Jeff Layton3471648a2015-07-10 15:58:42 -04002286 put_rpccred(data->cred);
Christoph Hellwig5f919c92014-08-21 11:09:25 -05002287 spin_lock(&inode->i_lock);
Trond Myklebust29559b12015-03-25 20:10:20 -04002288 set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags);
2289 if (end_pos > nfsi->layout->plh_lwb)
Christoph Hellwig5f919c92014-08-21 11:09:25 -05002290 nfsi->layout->plh_lwb = end_pos;
Jeff Layton3471648a2015-07-10 15:58:42 -04002291 goto out_unlock;
Christoph Hellwig5f919c92014-08-21 11:09:25 -05002292 }
2293 }
2294
2295
Andy Adamson863a3c62011-03-23 13:27:54 +00002296 status = nfs4_proc_layoutcommit(data, sync);
2297out:
Peng Tao92407e72011-10-23 20:21:17 -07002298 if (status)
2299 mark_inode_dirty_sync(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00002300 dprintk("<-- %s status %d\n", __func__, status);
2301 return status;
Trond Myklebust71244d92014-01-13 13:34:36 -05002302out_unlock:
2303 spin_unlock(&inode->i_lock);
Peng Tao92407e72011-10-23 20:21:17 -07002304 kfree(data);
Trond Myklebust71244d92014-01-13 13:34:36 -05002305clear_layoutcommitting:
2306 pnfs_clear_layoutcommitting(inode);
Peng Tao92407e72011-10-23 20:21:17 -07002307 goto out;
Andy Adamson863a3c62011-03-23 13:27:54 +00002308}
Peng Tao72cff442014-08-07 10:12:38 +08002309EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode);
Andy Adamson82be4172012-05-23 05:02:35 -04002310
Trond Myklebust5bb89b42015-03-25 14:14:42 -04002311int
2312pnfs_generic_sync(struct inode *inode, bool datasync)
2313{
2314 return pnfs_layoutcommit_inode(inode, true);
2315}
2316EXPORT_SYMBOL_GPL(pnfs_generic_sync);
2317
Andy Adamson82be4172012-05-23 05:02:35 -04002318struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
2319{
2320 struct nfs4_threshold *thp;
2321
2322 thp = kzalloc(sizeof(*thp), GFP_NOFS);
2323 if (!thp) {
2324 dprintk("%s mdsthreshold allocation failed\n", __func__);
2325 return NULL;
2326 }
2327 return thp;
2328}
Peng Tao87334082015-06-23 19:51:57 +08002329
Peng Tao865a7ec2015-06-25 18:19:32 +08002330#if IS_ENABLED(CONFIG_NFS_V4_2)
Peng Tao87334082015-06-23 19:51:57 +08002331int
Trond Myklebustc8ad8892015-08-05 17:31:58 -04002332pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags)
Peng Tao87334082015-06-23 19:51:57 +08002333{
2334 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
2335 struct nfs_server *server = NFS_SERVER(inode);
Peng Tao1bfe3b22015-06-23 19:52:03 +08002336 struct nfs_inode *nfsi = NFS_I(inode);
Peng Tao87334082015-06-23 19:51:57 +08002337 struct nfs42_layoutstat_data *data;
2338 struct pnfs_layout_hdr *hdr;
2339 int status = 0;
2340
2341 if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats)
2342 goto out;
2343
Trond Myklebust6c5a0d82015-06-27 11:45:46 -04002344 if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS))
2345 goto out;
2346
Peng Tao1bfe3b22015-06-23 19:52:03 +08002347 if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags))
2348 goto out;
2349
Peng Tao87334082015-06-23 19:51:57 +08002350 spin_lock(&inode->i_lock);
2351 if (!NFS_I(inode)->layout) {
2352 spin_unlock(&inode->i_lock);
2353 goto out;
2354 }
2355 hdr = NFS_I(inode)->layout;
2356 pnfs_get_layout_hdr(hdr);
2357 spin_unlock(&inode->i_lock);
2358
Trond Myklebustc8ad8892015-08-05 17:31:58 -04002359 data = kzalloc(sizeof(*data), gfp_flags);
Peng Tao87334082015-06-23 19:51:57 +08002360 if (!data) {
2361 status = -ENOMEM;
2362 goto out_put;
2363 }
2364
2365 data->args.fh = NFS_FH(inode);
2366 data->args.inode = inode;
2367 nfs4_stateid_copy(&data->args.stateid, &hdr->plh_stateid);
2368 status = ld->prepare_layoutstats(&data->args);
2369 if (status)
2370 goto out_free;
2371
2372 status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data);
2373
2374out:
2375 dprintk("%s returns %d\n", __func__, status);
2376 return status;
2377
2378out_free:
2379 kfree(data);
2380out_put:
2381 pnfs_put_layout_hdr(hdr);
Peng Tao1bfe3b22015-06-23 19:52:03 +08002382 smp_mb__before_atomic();
2383 clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags);
2384 smp_mb__after_atomic();
Peng Tao87334082015-06-23 19:51:57 +08002385 goto out;
2386}
2387EXPORT_SYMBOL_GPL(pnfs_report_layoutstat);
Peng Tao865a7ec2015-06-25 18:19:32 +08002388#endif
Trond Myklebustbbf58bf2015-08-24 20:39:18 -04002389
2390unsigned int layoutstats_timer;
2391module_param(layoutstats_timer, uint, 0644);
2392EXPORT_SYMBOL_GPL(layoutstats_timer);