blob: b7a07ba8783aa229a2dd124ac76ac4051253eaf1 [file] [log] [blame]
Ricardo Labiaga85e174b2010-10-20 00:17:58 -04001/*
2 * pNFS functions to call and manage layout drivers.
3 *
4 * Copyright (c) 2002 [year of first publication]
5 * The Regents of the University of Michigan
6 * All Rights Reserved
7 *
8 * Dean Hildebrand <dhildebz@umich.edu>
9 *
10 * Permission is granted to use, copy, create derivative works, and
11 * redistribute this software and such derivative works for any purpose,
12 * so long as the name of the University of Michigan is not used in
13 * any advertising or publicity pertaining to the use or distribution
14 * of this software without specific, written prior authorization. If
15 * the above copyright notice or any other identification of the
16 * University of Michigan is included in any copy of any portion of
17 * this software, then the disclaimer below must also be included.
18 *
19 * This software is provided as is, without representation or warranty
20 * of any kind either express or implied, including without limitation
21 * the implied warranties of merchantability, fitness for a particular
22 * purpose, or noninfringement. The Regents of the University of
23 * Michigan shall not be liable for any damages, including special,
24 * indirect, incidental, or consequential damages, with respect to any
25 * claim arising out of or in connection with the use of the software,
26 * even if it has been or is hereafter advised of the possibility of
27 * such damages.
28 */
29
30#include <linux/nfs_fs.h>
Trond Myklebust493292d2011-07-13 15:58:28 -040031#include <linux/nfs_page.h>
Paul Gortmaker143cb492011-07-01 14:23:34 -040032#include <linux/module.h>
Jeff Laytonca440c32016-09-15 14:40:49 -040033#include <linux/sort.h>
Andy Adamson974cec82010-10-20 00:18:02 -040034#include "internal.h"
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040035#include "pnfs.h"
Andy Adamson64419a92011-03-01 01:34:16 +000036#include "iostat.h"
Trond Myklebustcc668ab2013-08-14 15:31:28 -040037#include "nfs4trace.h"
Trond Myklebust40dd4b72015-01-24 13:54:37 -050038#include "delegation.h"
Peng Tao87334082015-06-23 19:51:57 +080039#include "nfs42.h"
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040040
41#define NFSDBG_FACILITY NFSDBG_PNFS
Trond Myklebust25c75332012-09-18 17:01:12 -040042#define PNFS_LAYOUTGET_RETRY_TIMEOUT (120*HZ)
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040043
Fred Isaman02c35fc2010-10-20 00:17:59 -040044/* Locking:
45 *
46 * pnfs_spinlock:
47 * protects pnfs_modules_tbl.
48 */
49static DEFINE_SPINLOCK(pnfs_spinlock);
50
51/*
52 * pnfs_modules_tbl holds all pnfs modules
53 */
54static LIST_HEAD(pnfs_modules_tbl);
55
Trond Myklebust13c13a62016-01-26 23:12:11 -050056static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo);
Peng Taoaa1e0e3a2014-09-06 00:53:25 +080057
Fred Isaman02c35fc2010-10-20 00:17:59 -040058/* Return the registered pnfs layout driver module matching given id */
59static struct pnfs_layoutdriver_type *
60find_pnfs_driver_locked(u32 id)
61{
62 struct pnfs_layoutdriver_type *local;
63
64 list_for_each_entry(local, &pnfs_modules_tbl, pnfs_tblid)
65 if (local->id == id)
66 goto out;
67 local = NULL;
68out:
69 dprintk("%s: Searching for id %u, found %p\n", __func__, id, local);
70 return local;
71}
72
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040073static struct pnfs_layoutdriver_type *
74find_pnfs_driver(u32 id)
75{
Fred Isaman02c35fc2010-10-20 00:17:59 -040076 struct pnfs_layoutdriver_type *local;
77
78 spin_lock(&pnfs_spinlock);
79 local = find_pnfs_driver_locked(id);
Trond Myklebust0a9c63f2012-06-15 13:02:58 -040080 if (local != NULL && !try_module_get(local->owner)) {
81 dprintk("%s: Could not grab reference on module\n", __func__);
82 local = NULL;
83 }
Fred Isaman02c35fc2010-10-20 00:17:59 -040084 spin_unlock(&pnfs_spinlock);
85 return local;
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040086}
87
88void
89unset_pnfs_layoutdriver(struct nfs_server *nfss)
90{
Benny Halevy738fd0f32011-07-30 20:52:36 -040091 if (nfss->pnfs_curr_ld) {
92 if (nfss->pnfs_curr_ld->clear_layoutdriver)
93 nfss->pnfs_curr_ld->clear_layoutdriver(nfss);
Trond Myklebust2a4c8992012-06-14 13:08:38 -040094 /* Decrement the MDS count. Purge the deviceid cache if zero */
95 if (atomic_dec_and_test(&nfss->nfs_client->cl_mds_count))
96 nfs4_deviceid_purge_client(nfss->nfs_client);
Fred Isaman02c35fc2010-10-20 00:17:59 -040097 module_put(nfss->pnfs_curr_ld->owner);
Benny Halevy738fd0f32011-07-30 20:52:36 -040098 }
Ricardo Labiaga85e174b2010-10-20 00:17:58 -040099 nfss->pnfs_curr_ld = NULL;
100}
101
102/*
Jeff Laytonca440c32016-09-15 14:40:49 -0400103 * When the server sends a list of layout types, we choose one in the order
104 * given in the list below.
105 *
106 * FIXME: should this list be configurable in some fashion? module param?
107 * mount option? something else?
108 */
109static const u32 ld_prefs[] = {
110 LAYOUT_SCSI,
111 LAYOUT_BLOCK_VOLUME,
112 LAYOUT_OSD2_OBJECTS,
113 LAYOUT_FLEX_FILES,
114 LAYOUT_NFSV4_1_FILES,
115 0
116};
117
118static int
119ld_cmp(const void *e1, const void *e2)
120{
121 u32 ld1 = *((u32 *)e1);
122 u32 ld2 = *((u32 *)e2);
123 int i;
124
125 for (i = 0; ld_prefs[i] != 0; i++) {
126 if (ld1 == ld_prefs[i])
127 return -1;
128
129 if (ld2 == ld_prefs[i])
130 return 1;
131 }
132 return 0;
133}
134
135/*
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400136 * Try to set the server's pnfs module to the pnfs layout type specified by id.
137 * Currently only one pNFS layout driver per filesystem is supported.
138 *
Jeff Layton3132e492016-08-10 15:58:24 -0400139 * @ids array of layout types supported by MDS.
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400140 */
141void
Benny Halevy738fd0f32011-07-30 20:52:36 -0400142set_pnfs_layoutdriver(struct nfs_server *server, const struct nfs_fh *mntfh,
Jeff Laytonca440c32016-09-15 14:40:49 -0400143 struct nfs_fsinfo *fsinfo)
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400144{
145 struct pnfs_layoutdriver_type *ld_type = NULL;
Jeff Layton3132e492016-08-10 15:58:24 -0400146 u32 id;
Jeff Laytonca440c32016-09-15 14:40:49 -0400147 int i;
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400148
Anna Schumaker19274712016-10-26 15:54:31 -0400149 if (fsinfo->nlayouttypes == 0)
150 goto out_no_driver;
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400151 if (!(server->nfs_client->cl_exchange_flags &
152 (EXCHGID4_FLAG_USE_NON_PNFS | EXCHGID4_FLAG_USE_PNFS_MDS))) {
Jeff Layton3132e492016-08-10 15:58:24 -0400153 printk(KERN_ERR "NFS: %s: cl_exchange_flags 0x%x\n",
154 __func__, server->nfs_client->cl_exchange_flags);
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400155 goto out_no_driver;
156 }
Jeff Layton3132e492016-08-10 15:58:24 -0400157
Jeff Laytonca440c32016-09-15 14:40:49 -0400158 sort(fsinfo->layouttype, fsinfo->nlayouttypes,
159 sizeof(*fsinfo->layouttype), ld_cmp, NULL);
Jeff Layton3132e492016-08-10 15:58:24 -0400160
Jeff Laytonca440c32016-09-15 14:40:49 -0400161 for (i = 0; i < fsinfo->nlayouttypes; i++) {
162 id = fsinfo->layouttype[i];
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400163 ld_type = find_pnfs_driver(id);
Jeff Laytonca440c32016-09-15 14:40:49 -0400164 if (!ld_type) {
165 request_module("%s-%u", LAYOUT_NFSV4_1_MODULE_PREFIX,
166 id);
167 ld_type = find_pnfs_driver(id);
168 }
169 if (ld_type)
170 break;
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400171 }
Jeff Layton3132e492016-08-10 15:58:24 -0400172
173 if (!ld_type) {
Jeff Laytonca440c32016-09-15 14:40:49 -0400174 dprintk("%s: No pNFS module found!\n", __func__);
Jeff Layton3132e492016-08-10 15:58:24 -0400175 goto out_no_driver;
176 }
177
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400178 server->pnfs_curr_ld = ld_type;
Benny Halevy738fd0f32011-07-30 20:52:36 -0400179 if (ld_type->set_layoutdriver
180 && ld_type->set_layoutdriver(server, mntfh)) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500181 printk(KERN_ERR "NFS: %s: Error initializing pNFS layout "
182 "driver %u.\n", __func__, id);
Benny Halevy738fd0f32011-07-30 20:52:36 -0400183 module_put(ld_type->owner);
184 goto out_no_driver;
185 }
Trond Myklebust2a4c8992012-06-14 13:08:38 -0400186 /* Bump the MDS count */
187 atomic_inc(&server->nfs_client->cl_mds_count);
Christoph Hellwigea8eecd2011-03-01 01:34:21 +0000188
Ricardo Labiaga85e174b2010-10-20 00:17:58 -0400189 dprintk("%s: pNFS module for %u set\n", __func__, id);
190 return;
191
192out_no_driver:
193 dprintk("%s: Using NFSv4 I/O\n", __func__);
194 server->pnfs_curr_ld = NULL;
195}
Fred Isaman02c35fc2010-10-20 00:17:59 -0400196
197int
198pnfs_register_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
199{
200 int status = -EINVAL;
201 struct pnfs_layoutdriver_type *tmp;
202
203 if (ld_type->id == 0) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500204 printk(KERN_ERR "NFS: %s id 0 is reserved\n", __func__);
Fred Isaman02c35fc2010-10-20 00:17:59 -0400205 return status;
206 }
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400207 if (!ld_type->alloc_lseg || !ld_type->free_lseg) {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500208 printk(KERN_ERR "NFS: %s Layout driver must provide "
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400209 "alloc_lseg and free_lseg.\n", __func__);
210 return status;
211 }
Fred Isaman02c35fc2010-10-20 00:17:59 -0400212
213 spin_lock(&pnfs_spinlock);
214 tmp = find_pnfs_driver_locked(ld_type->id);
215 if (!tmp) {
216 list_add(&ld_type->pnfs_tblid, &pnfs_modules_tbl);
217 status = 0;
218 dprintk("%s Registering id:%u name:%s\n", __func__, ld_type->id,
219 ld_type->name);
220 } else {
Weston Andros Adamsona0308892012-01-26 13:32:23 -0500221 printk(KERN_ERR "NFS: %s Module with id %d already loaded!\n",
Fred Isaman02c35fc2010-10-20 00:17:59 -0400222 __func__, ld_type->id);
223 }
224 spin_unlock(&pnfs_spinlock);
225
226 return status;
227}
228EXPORT_SYMBOL_GPL(pnfs_register_layoutdriver);
229
230void
231pnfs_unregister_layoutdriver(struct pnfs_layoutdriver_type *ld_type)
232{
233 dprintk("%s Deregistering id:%u\n", __func__, ld_type->id);
234 spin_lock(&pnfs_spinlock);
235 list_del(&ld_type->pnfs_tblid);
236 spin_unlock(&pnfs_spinlock);
237}
238EXPORT_SYMBOL_GPL(pnfs_unregister_layoutdriver);
Benny Halevye5e94012010-10-20 00:18:01 -0400239
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400240/*
241 * pNFS client layout cache
242 */
243
Fred Isamancc6e5342011-01-06 11:36:28 +0000244/* Need to hold i_lock if caller does not already hold reference */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000245void
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400246pnfs_get_layout_hdr(struct pnfs_layout_hdr *lo)
Benny Halevye5e94012010-10-20 00:18:01 -0400247{
Fred Isamancc6e5342011-01-06 11:36:28 +0000248 atomic_inc(&lo->plh_refcount);
249}
250
Benny Halevy636fb9c2011-05-22 19:51:33 +0300251static struct pnfs_layout_hdr *
252pnfs_alloc_layout_hdr(struct inode *ino, gfp_t gfp_flags)
253{
254 struct pnfs_layoutdriver_type *ld = NFS_SERVER(ino)->pnfs_curr_ld;
Trond Myklebust57934272012-09-20 20:37:23 -0400255 return ld->alloc_layout_hdr(ino, gfp_flags);
Benny Halevy636fb9c2011-05-22 19:51:33 +0300256}
257
258static void
259pnfs_free_layout_hdr(struct pnfs_layout_hdr *lo)
260{
Trond Myklebust9c626382012-09-20 17:31:43 -0400261 struct nfs_server *server = NFS_SERVER(lo->plh_inode);
262 struct pnfs_layoutdriver_type *ld = server->pnfs_curr_ld;
263
264 if (!list_empty(&lo->plh_layouts)) {
265 struct nfs_client *clp = server->nfs_client;
266
267 spin_lock(&clp->cl_lock);
268 list_del_init(&lo->plh_layouts);
269 spin_unlock(&clp->cl_lock);
270 }
Peng Tao9fa40752011-07-30 20:52:32 -0400271 put_rpccred(lo->plh_lc_cred);
Trond Myklebust57934272012-09-20 20:37:23 -0400272 return ld->free_layout_hdr(lo);
Benny Halevy636fb9c2011-05-22 19:51:33 +0300273}
274
Fred Isamancc6e5342011-01-06 11:36:28 +0000275static void
Trond Myklebust6622c3e2012-09-20 17:23:11 -0400276pnfs_detach_layout_hdr(struct pnfs_layout_hdr *lo)
Fred Isamancc6e5342011-01-06 11:36:28 +0000277{
Trond Myklebustbb346f62012-09-20 15:52:13 -0400278 struct nfs_inode *nfsi = NFS_I(lo->plh_inode);
Fred Isamancc6e5342011-01-06 11:36:28 +0000279 dprintk("%s: freeing layout cache %p\n", __func__, lo);
Trond Myklebustbb346f62012-09-20 15:52:13 -0400280 nfsi->layout = NULL;
281 /* Reset MDS Threshold I/O counters */
282 nfsi->write_io = 0;
283 nfsi->read_io = 0;
Benny Halevye5e94012010-10-20 00:18:01 -0400284}
285
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400286void
Trond Myklebust70c3bd22012-09-18 20:51:13 -0400287pnfs_put_layout_hdr(struct pnfs_layout_hdr *lo)
Andy Adamson974cec82010-10-20 00:18:02 -0400288{
Fred Isamancc6e5342011-01-06 11:36:28 +0000289 struct inode *inode = lo->plh_inode;
290
Trond Myklebust13c13a62016-01-26 23:12:11 -0500291 pnfs_layoutreturn_before_put_layout_hdr(lo);
292
Fred Isamancc6e5342011-01-06 11:36:28 +0000293 if (atomic_dec_and_lock(&lo->plh_refcount, &inode->i_lock)) {
Peng Tao566f8732014-10-10 23:25:46 +0800294 if (!list_empty(&lo->plh_segs))
295 WARN_ONCE(1, "NFS: BUG unfreed layout segments.\n");
Trond Myklebust6622c3e2012-09-20 17:23:11 -0400296 pnfs_detach_layout_hdr(lo);
Fred Isamancc6e5342011-01-06 11:36:28 +0000297 spin_unlock(&inode->i_lock);
Trond Myklebust6622c3e2012-09-20 17:23:11 -0400298 pnfs_free_layout_hdr(lo);
Fred Isamancc6e5342011-01-06 11:36:28 +0000299 }
Andy Adamson974cec82010-10-20 00:18:02 -0400300}
301
Trond Myklebuste523ce82016-11-14 14:34:18 -0500302static void
303pnfs_clear_layoutreturn_info(struct pnfs_layout_hdr *lo)
304{
305 lo->plh_return_iomode = 0;
306 lo->plh_return_seq = 0;
307 clear_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
308}
309
Trond Myklebust2454dfe2016-02-22 17:34:59 -0500310/*
311 * Mark a pnfs_layout_hdr and all associated layout segments as invalid
312 *
313 * In order to continue using the pnfs_layout_hdr, a full recovery
314 * is required.
315 * Note that caller must hold inode->i_lock.
316 */
Trond Myklebust5f46be02016-07-22 11:25:27 -0400317int
Trond Myklebust2454dfe2016-02-22 17:34:59 -0500318pnfs_mark_layout_stateid_invalid(struct pnfs_layout_hdr *lo,
319 struct list_head *lseg_list)
320{
321 struct pnfs_layout_range range = {
322 .iomode = IOMODE_ANY,
323 .offset = 0,
324 .length = NFS4_MAX_UINT64,
325 };
326
327 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
Trond Myklebuste523ce82016-11-14 14:34:18 -0500328 pnfs_clear_layoutreturn_info(lo);
Jeff Layton6d597e12016-05-17 12:28:42 -0400329 return pnfs_mark_matching_lsegs_invalid(lo, lseg_list, &range, 0);
Trond Myklebust2454dfe2016-02-22 17:34:59 -0500330}
331
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400332static int
333pnfs_iomode_to_fail_bit(u32 iomode)
334{
335 return iomode == IOMODE_RW ?
336 NFS_LAYOUT_RW_FAILED : NFS_LAYOUT_RO_FAILED;
337}
338
339static void
Trond Myklebust3e621212012-09-24 13:07:16 -0400340pnfs_layout_set_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400341{
Trond Myklebust25c75332012-09-18 17:01:12 -0400342 lo->plh_retry_timestamp = jiffies;
Yanchuan Nian39e88fc2013-01-04 20:19:49 +0800343 if (!test_and_set_bit(fail_bit, &lo->plh_flags))
Trond Myklebust3e621212012-09-24 13:07:16 -0400344 atomic_inc(&lo->plh_refcount);
345}
346
347static void
348pnfs_layout_clear_fail_bit(struct pnfs_layout_hdr *lo, int fail_bit)
349{
350 if (test_and_clear_bit(fail_bit, &lo->plh_flags))
351 atomic_dec(&lo->plh_refcount);
352}
353
354static void
355pnfs_layout_io_set_failed(struct pnfs_layout_hdr *lo, u32 iomode)
356{
357 struct inode *inode = lo->plh_inode;
Trond Myklebust115ce572012-09-20 21:19:43 -0400358 struct pnfs_layout_range range = {
359 .iomode = iomode,
360 .offset = 0,
361 .length = NFS4_MAX_UINT64,
362 };
363 LIST_HEAD(head);
Trond Myklebust3e621212012-09-24 13:07:16 -0400364
365 spin_lock(&inode->i_lock);
366 pnfs_layout_set_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
Jeff Layton6d597e12016-05-17 12:28:42 -0400367 pnfs_mark_matching_lsegs_invalid(lo, &head, &range, 0);
Trond Myklebust3e621212012-09-24 13:07:16 -0400368 spin_unlock(&inode->i_lock);
Trond Myklebust115ce572012-09-20 21:19:43 -0400369 pnfs_free_lseg_list(&head);
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400370 dprintk("%s Setting layout IOMODE_%s fail bit\n", __func__,
371 iomode == IOMODE_RW ? "RW" : "READ");
372}
373
374static bool
375pnfs_layout_io_test_failed(struct pnfs_layout_hdr *lo, u32 iomode)
376{
Trond Myklebust25c75332012-09-18 17:01:12 -0400377 unsigned long start, end;
Trond Myklebust3e621212012-09-24 13:07:16 -0400378 int fail_bit = pnfs_iomode_to_fail_bit(iomode);
379
380 if (test_bit(fail_bit, &lo->plh_flags) == 0)
Trond Myklebust25c75332012-09-18 17:01:12 -0400381 return false;
382 end = jiffies;
383 start = end - PNFS_LAYOUTGET_RETRY_TIMEOUT;
384 if (!time_in_range(lo->plh_retry_timestamp, start, end)) {
385 /* It is time to retry the failed layoutgets */
Trond Myklebust3e621212012-09-24 13:07:16 -0400386 pnfs_layout_clear_fail_bit(lo, fail_bit);
Trond Myklebust25c75332012-09-18 17:01:12 -0400387 return false;
388 }
389 return true;
Trond Myklebustb9e028f2012-09-18 16:41:18 -0400390}
391
Andy Adamson974cec82010-10-20 00:18:02 -0400392static void
Trond Myklebust119cef972016-07-24 15:10:12 -0400393pnfs_init_lseg(struct pnfs_layout_hdr *lo, struct pnfs_layout_segment *lseg,
394 const struct pnfs_layout_range *range,
395 const nfs4_stateid *stateid)
Andy Adamson974cec82010-10-20 00:18:02 -0400396{
Fred Isaman566052c2011-01-06 11:36:20 +0000397 INIT_LIST_HEAD(&lseg->pls_list);
Peng Taoa9bae562011-07-30 20:52:33 -0400398 INIT_LIST_HEAD(&lseg->pls_lc_list);
Fred Isaman4541d162011-01-06 11:36:23 +0000399 atomic_set(&lseg->pls_refcount, 1);
Fred Isaman4541d162011-01-06 11:36:23 +0000400 set_bit(NFS_LSEG_VALID, &lseg->pls_flags);
Fred Isaman566052c2011-01-06 11:36:20 +0000401 lseg->pls_layout = lo;
Trond Myklebust119cef972016-07-24 15:10:12 -0400402 lseg->pls_range = *range;
403 lseg->pls_seq = be32_to_cpu(stateid->seqid);
Andy Adamson974cec82010-10-20 00:18:02 -0400404}
405
Trond Myklebust905ca192012-09-20 20:46:49 -0400406static void pnfs_free_lseg(struct pnfs_layout_segment *lseg)
Andy Adamson974cec82010-10-20 00:18:02 -0400407{
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000408 struct inode *ino = lseg->pls_layout->plh_inode;
Andy Adamson974cec82010-10-20 00:18:02 -0400409
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400410 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
Andy Adamson974cec82010-10-20 00:18:02 -0400411}
412
Fred Isamand684d2a2011-03-01 01:34:13 +0000413static void
Trond Myklebust57036a32012-09-20 16:33:30 -0400414pnfs_layout_remove_lseg(struct pnfs_layout_hdr *lo,
415 struct pnfs_layout_segment *lseg)
Andy Adamson974cec82010-10-20 00:18:02 -0400416{
Trond Myklebust57036a32012-09-20 16:33:30 -0400417 struct inode *inode = lo->plh_inode;
Fred Isamand684d2a2011-03-01 01:34:13 +0000418
Benny Halevyd20581a2011-05-22 19:52:03 +0300419 WARN_ON(test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
Fred Isamand684d2a2011-03-01 01:34:13 +0000420 list_del_init(&lseg->pls_list);
Trond Myklebust8f0d27d2012-09-20 20:57:11 -0400421 /* Matched by pnfs_get_layout_hdr in pnfs_layout_insert_lseg */
422 atomic_dec(&lo->plh_refcount);
Trond Myklebustf32659d2016-11-14 13:10:48 -0500423 if (list_empty(&lo->plh_segs) &&
424 !test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags) &&
425 !test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
Trond Myklebust334a8f32016-09-04 12:46:35 -0400426 if (atomic_read(&lo->plh_outstanding) == 0)
427 set_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
Trond Myklebust173f77e2012-09-21 15:49:42 -0400428 clear_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
Trond Myklebust2d148c72016-06-17 16:48:26 -0400429 }
Fred Isamand684d2a2011-03-01 01:34:13 +0000430 rpc_wake_up(&NFS_SERVER(inode)->roc_rpcwaitq);
431}
432
Fred Isamanbae724e2011-03-01 01:34:15 +0000433void
Trond Myklebust9369a432012-09-18 20:57:08 -0400434pnfs_put_lseg(struct pnfs_layout_segment *lseg)
Fred Isamand684d2a2011-03-01 01:34:13 +0000435{
Trond Myklebust57036a32012-09-20 16:33:30 -0400436 struct pnfs_layout_hdr *lo;
Fred Isamand684d2a2011-03-01 01:34:13 +0000437 struct inode *inode;
438
439 if (!lseg)
440 return;
441
Fred Isaman4541d162011-01-06 11:36:23 +0000442 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
443 atomic_read(&lseg->pls_refcount),
444 test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
Trond Myklebust4ef2e4f2015-02-05 17:27:39 -0500445
Trond Myklebust57036a32012-09-20 16:33:30 -0400446 lo = lseg->pls_layout;
447 inode = lo->plh_inode;
Trond Myklebust4ef2e4f2015-02-05 17:27:39 -0500448
Fred Isamand684d2a2011-03-01 01:34:13 +0000449 if (atomic_dec_and_lock(&lseg->pls_refcount, &inode->i_lock)) {
Trond Myklebustfaa4a542015-07-09 18:24:07 +0200450 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
451 spin_unlock(&inode->i_lock);
452 return;
453 }
Trond Myklebust8f0d27d2012-09-20 20:57:11 -0400454 pnfs_get_layout_hdr(lo);
Trond Myklebust4ef2e4f2015-02-05 17:27:39 -0500455 pnfs_layout_remove_lseg(lo, lseg);
456 spin_unlock(&inode->i_lock);
457 pnfs_free_lseg(lseg);
458 pnfs_put_layout_hdr(lo);
Fred Isaman4541d162011-01-06 11:36:23 +0000459 }
Andy Adamson974cec82010-10-20 00:18:02 -0400460}
Trond Myklebust9369a432012-09-18 20:57:08 -0400461EXPORT_SYMBOL_GPL(pnfs_put_lseg);
Andy Adamson974cec82010-10-20 00:18:02 -0400462
Trond Myklebust6543f802014-10-08 16:39:12 -0400463static void pnfs_free_lseg_async_work(struct work_struct *work)
Weston Andros Adamsone6cf82d2014-07-17 20:42:18 -0400464{
465 struct pnfs_layout_segment *lseg;
Trond Myklebust6543f802014-10-08 16:39:12 -0400466 struct pnfs_layout_hdr *lo;
Weston Andros Adamsone6cf82d2014-07-17 20:42:18 -0400467
468 lseg = container_of(work, struct pnfs_layout_segment, pls_work);
Trond Myklebust6543f802014-10-08 16:39:12 -0400469 lo = lseg->pls_layout;
Weston Andros Adamsone6cf82d2014-07-17 20:42:18 -0400470
Trond Myklebust6543f802014-10-08 16:39:12 -0400471 pnfs_free_lseg(lseg);
472 pnfs_put_layout_hdr(lo);
473}
474
475static void pnfs_free_lseg_async(struct pnfs_layout_segment *lseg)
476{
477 INIT_WORK(&lseg->pls_work, pnfs_free_lseg_async_work);
478 schedule_work(&lseg->pls_work);
Weston Andros Adamsone6cf82d2014-07-17 20:42:18 -0400479}
480
481void
Trond Myklebust6543f802014-10-08 16:39:12 -0400482pnfs_put_lseg_locked(struct pnfs_layout_segment *lseg)
Weston Andros Adamsone6cf82d2014-07-17 20:42:18 -0400483{
Trond Myklebust6543f802014-10-08 16:39:12 -0400484 if (!lseg)
485 return;
486
487 assert_spin_locked(&lseg->pls_layout->plh_inode->i_lock);
488
489 dprintk("%s: lseg %p ref %d valid %d\n", __func__, lseg,
490 atomic_read(&lseg->pls_refcount),
491 test_bit(NFS_LSEG_VALID, &lseg->pls_flags));
492 if (atomic_dec_and_test(&lseg->pls_refcount)) {
493 struct pnfs_layout_hdr *lo = lseg->pls_layout;
Trond Myklebustfaa4a542015-07-09 18:24:07 +0200494 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags))
495 return;
Trond Myklebust6543f802014-10-08 16:39:12 -0400496 pnfs_get_layout_hdr(lo);
497 pnfs_layout_remove_lseg(lo, lseg);
498 pnfs_free_lseg_async(lseg);
499 }
Weston Andros Adamsone6cf82d2014-07-17 20:42:18 -0400500}
Trond Myklebust6543f802014-10-08 16:39:12 -0400501EXPORT_SYMBOL_GPL(pnfs_put_lseg_locked);
Weston Andros Adamsone6cf82d2014-07-17 20:42:18 -0400502
Trond Myklebust3cb2df12013-06-03 11:24:36 -0400503static u64
Benny Halevyfb3296e2011-05-22 19:47:26 +0300504end_offset(u64 start, u64 len)
Fred Isaman4541d162011-01-06 11:36:23 +0000505{
Benny Halevyfb3296e2011-05-22 19:47:26 +0300506 u64 end;
507
508 end = start + len;
509 return end >= start ? end : NFS4_MAX_UINT64;
510}
511
Benny Halevyfb3296e2011-05-22 19:47:26 +0300512/*
513 * is l2 fully contained in l1?
514 * start1 end1
515 * [----------------------------------)
516 * start2 end2
517 * [----------------)
518 */
Trond Myklebust3cb2df12013-06-03 11:24:36 -0400519static bool
Trond Myklebust7dc0ac72013-06-03 11:30:24 -0400520pnfs_lseg_range_contained(const struct pnfs_layout_range *l1,
Trond Myklebust3cb2df12013-06-03 11:24:36 -0400521 const struct pnfs_layout_range *l2)
Benny Halevyfb3296e2011-05-22 19:47:26 +0300522{
523 u64 start1 = l1->offset;
524 u64 end1 = end_offset(start1, l1->length);
525 u64 start2 = l2->offset;
526 u64 end2 = end_offset(start2, l2->length);
527
528 return (start1 <= start2) && (end1 >= end2);
529}
530
531/*
532 * is l1 and l2 intersecting?
533 * start1 end1
534 * [----------------------------------)
535 * start2 end2
536 * [----------------)
537 */
Trond Myklebust3cb2df12013-06-03 11:24:36 -0400538static bool
Trond Myklebust7dc0ac72013-06-03 11:30:24 -0400539pnfs_lseg_range_intersecting(const struct pnfs_layout_range *l1,
Trond Myklebust3cb2df12013-06-03 11:24:36 -0400540 const struct pnfs_layout_range *l2)
Benny Halevyfb3296e2011-05-22 19:47:26 +0300541{
542 u64 start1 = l1->offset;
543 u64 end1 = end_offset(start1, l1->length);
544 u64 start2 = l2->offset;
545 u64 end2 = end_offset(start2, l2->length);
546
547 return (end1 == NFS4_MAX_UINT64 || end1 > start2) &&
548 (end2 == NFS4_MAX_UINT64 || end2 > start1);
549}
550
Trond Myklebust24956802013-03-20 13:03:00 -0400551static bool pnfs_lseg_dec_and_remove_zero(struct pnfs_layout_segment *lseg,
552 struct list_head *tmp_list)
553{
554 if (!atomic_dec_and_test(&lseg->pls_refcount))
555 return false;
556 pnfs_layout_remove_lseg(lseg->pls_layout, lseg);
557 list_add(&lseg->pls_list, tmp_list);
558 return true;
559}
560
Fred Isaman4541d162011-01-06 11:36:23 +0000561/* Returns 1 if lseg is removed from list, 0 otherwise */
562static int mark_lseg_invalid(struct pnfs_layout_segment *lseg,
563 struct list_head *tmp_list)
564{
565 int rv = 0;
566
567 if (test_and_clear_bit(NFS_LSEG_VALID, &lseg->pls_flags)) {
568 /* Remove the reference keeping the lseg in the
569 * list. It will now be removed when all
570 * outstanding io is finished.
571 */
Fred Isamand684d2a2011-03-01 01:34:13 +0000572 dprintk("%s: lseg %p ref %d\n", __func__, lseg,
573 atomic_read(&lseg->pls_refcount));
Trond Myklebust24956802013-03-20 13:03:00 -0400574 if (pnfs_lseg_dec_and_remove_zero(lseg, tmp_list))
Fred Isamand684d2a2011-03-01 01:34:13 +0000575 rv = 1;
Fred Isaman4541d162011-01-06 11:36:23 +0000576 }
577 return rv;
578}
579
Jeff Layton6d597e12016-05-17 12:28:42 -0400580/*
581 * Compare 2 layout stateid sequence ids, to see which is newer,
582 * taking into account wraparound issues.
583 */
584static bool pnfs_seqid_is_newer(u32 s1, u32 s2)
585{
586 return (s32)(s1 - s2) > 0;
587}
588
Trond Myklebuste036f462016-07-22 11:13:22 -0400589static bool
590pnfs_should_free_range(const struct pnfs_layout_range *lseg_range,
591 const struct pnfs_layout_range *recall_range)
592{
593 return (recall_range->iomode == IOMODE_ANY ||
594 lseg_range->iomode == recall_range->iomode) &&
595 pnfs_lseg_range_intersecting(lseg_range, recall_range);
596}
597
598static bool
599pnfs_match_lseg_recall(const struct pnfs_layout_segment *lseg,
600 const struct pnfs_layout_range *recall_range,
601 u32 seq)
602{
603 if (seq != 0 && pnfs_seqid_is_newer(lseg->pls_seq, seq))
604 return false;
605 if (recall_range == NULL)
606 return true;
607 return pnfs_should_free_range(&lseg->pls_range, recall_range);
608}
609
Jeff Layton6d597e12016-05-17 12:28:42 -0400610/**
611 * pnfs_mark_matching_lsegs_invalid - tear down lsegs or mark them for later
612 * @lo: layout header containing the lsegs
613 * @tmp_list: list head where doomed lsegs should go
614 * @recall_range: optional recall range argument to match (may be NULL)
615 * @seq: only invalidate lsegs obtained prior to this sequence (may be 0)
616 *
617 * Walk the list of lsegs in the layout header, and tear down any that should
618 * be destroyed. If "recall_range" is specified then the segment must match
619 * that range. If "seq" is non-zero, then only match segments that were handed
620 * out at or before that sequence.
621 *
622 * Returns number of matching invalid lsegs remaining in list after scanning
623 * it and purging them.
Fred Isaman4541d162011-01-06 11:36:23 +0000624 */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000625int
Trond Myklebust49a85062012-09-18 20:43:31 -0400626pnfs_mark_matching_lsegs_invalid(struct pnfs_layout_hdr *lo,
Fred Isaman4541d162011-01-06 11:36:23 +0000627 struct list_head *tmp_list,
Jeff Layton6d597e12016-05-17 12:28:42 -0400628 const struct pnfs_layout_range *recall_range,
629 u32 seq)
Andy Adamson974cec82010-10-20 00:18:02 -0400630{
631 struct pnfs_layout_segment *lseg, *next;
Trond Myklebust71b39852016-01-04 12:41:15 -0500632 int remaining = 0;
Andy Adamson974cec82010-10-20 00:18:02 -0400633
634 dprintk("%s:Begin lo %p\n", __func__, lo);
635
Trond Myklebust8006bfb2012-09-21 14:48:04 -0400636 if (list_empty(&lo->plh_segs))
Fred Isaman38511722011-02-03 18:28:50 +0000637 return 0;
Fred Isaman4541d162011-01-06 11:36:23 +0000638 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
Trond Myklebuste036f462016-07-22 11:13:22 -0400639 if (pnfs_match_lseg_recall(lseg, recall_range, seq)) {
Jeff Layton6d597e12016-05-17 12:28:42 -0400640 dprintk("%s: freeing lseg %p iomode %d seq %u"
Fred Isaman4541d162011-01-06 11:36:23 +0000641 "offset %llu length %llu\n", __func__,
Jeff Layton6d597e12016-05-17 12:28:42 -0400642 lseg, lseg->pls_range.iomode, lseg->pls_seq,
643 lseg->pls_range.offset, lseg->pls_range.length);
Trond Myklebust71b39852016-01-04 12:41:15 -0500644 if (!mark_lseg_invalid(lseg, tmp_list))
645 remaining++;
Fred Isaman4541d162011-01-06 11:36:23 +0000646 }
Trond Myklebust71b39852016-01-04 12:41:15 -0500647 dprintk("%s:Return %i\n", __func__, remaining);
648 return remaining;
Andy Adamson974cec82010-10-20 00:18:02 -0400649}
650
Fred Isamanf49f9ba2011-02-03 18:28:52 +0000651/* note free_me must contain lsegs from a single layout_hdr */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000652void
Fred Isaman4541d162011-01-06 11:36:23 +0000653pnfs_free_lseg_list(struct list_head *free_me)
Andy Adamson974cec82010-10-20 00:18:02 -0400654{
Fred Isaman4541d162011-01-06 11:36:23 +0000655 struct pnfs_layout_segment *lseg, *tmp;
Andy Adamson974cec82010-10-20 00:18:02 -0400656
Fred Isamanf49f9ba2011-02-03 18:28:52 +0000657 if (list_empty(free_me))
658 return;
659
Fred Isaman4541d162011-01-06 11:36:23 +0000660 list_for_each_entry_safe(lseg, tmp, free_me, pls_list) {
Fred Isaman566052c2011-01-06 11:36:20 +0000661 list_del(&lseg->pls_list);
Trond Myklebust905ca192012-09-20 20:46:49 -0400662 pnfs_free_lseg(lseg);
Andy Adamson974cec82010-10-20 00:18:02 -0400663 }
664}
665
Benny Halevye5e94012010-10-20 00:18:01 -0400666void
667pnfs_destroy_layout(struct nfs_inode *nfsi)
668{
669 struct pnfs_layout_hdr *lo;
Andy Adamson974cec82010-10-20 00:18:02 -0400670 LIST_HEAD(tmp_list);
Benny Halevye5e94012010-10-20 00:18:01 -0400671
672 spin_lock(&nfsi->vfs_inode.i_lock);
673 lo = nfsi->layout;
674 if (lo) {
Trond Myklebust3e621212012-09-24 13:07:16 -0400675 pnfs_get_layout_hdr(lo);
Trond Myklebust2454dfe2016-02-22 17:34:59 -0500676 pnfs_mark_layout_stateid_invalid(lo, &tmp_list);
Trond Myklebust3e621212012-09-24 13:07:16 -0400677 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RO_FAILED);
678 pnfs_layout_clear_fail_bit(lo, NFS_LAYOUT_RW_FAILED);
679 spin_unlock(&nfsi->vfs_inode.i_lock);
680 pnfs_free_lseg_list(&tmp_list);
681 pnfs_put_layout_hdr(lo);
682 } else
683 spin_unlock(&nfsi->vfs_inode.i_lock);
Benny Halevye5e94012010-10-20 00:18:01 -0400684}
Andy Adamson041245c2012-04-27 17:53:53 -0400685EXPORT_SYMBOL_GPL(pnfs_destroy_layout);
Benny Halevye5e94012010-10-20 00:18:01 -0400686
Trond Myklebustfd9a8d72013-02-12 09:48:42 -0500687static bool
688pnfs_layout_add_bulk_destroy_list(struct inode *inode,
689 struct list_head *layout_list)
690{
691 struct pnfs_layout_hdr *lo;
692 bool ret = false;
693
694 spin_lock(&inode->i_lock);
695 lo = NFS_I(inode)->layout;
696 if (lo != NULL && list_empty(&lo->plh_bulk_destroy)) {
697 pnfs_get_layout_hdr(lo);
698 list_add(&lo->plh_bulk_destroy, layout_list);
699 ret = true;
700 }
701 spin_unlock(&inode->i_lock);
702 return ret;
703}
704
705/* Caller must hold rcu_read_lock and clp->cl_lock */
706static int
707pnfs_layout_bulk_destroy_byserver_locked(struct nfs_client *clp,
708 struct nfs_server *server,
709 struct list_head *layout_list)
710{
711 struct pnfs_layout_hdr *lo, *next;
712 struct inode *inode;
713
714 list_for_each_entry_safe(lo, next, &server->layouts, plh_layouts) {
715 inode = igrab(lo->plh_inode);
716 if (inode == NULL)
717 continue;
718 list_del_init(&lo->plh_layouts);
719 if (pnfs_layout_add_bulk_destroy_list(inode, layout_list))
720 continue;
721 rcu_read_unlock();
722 spin_unlock(&clp->cl_lock);
723 iput(inode);
724 spin_lock(&clp->cl_lock);
725 rcu_read_lock();
726 return -EAGAIN;
727 }
728 return 0;
729}
730
731static int
732pnfs_layout_free_bulk_destroy_list(struct list_head *layout_list,
733 bool is_bulk_recall)
734{
735 struct pnfs_layout_hdr *lo;
736 struct inode *inode;
Trond Myklebustfd9a8d72013-02-12 09:48:42 -0500737 LIST_HEAD(lseg_list);
738 int ret = 0;
739
740 while (!list_empty(layout_list)) {
741 lo = list_entry(layout_list->next, struct pnfs_layout_hdr,
742 plh_bulk_destroy);
743 dprintk("%s freeing layout for inode %lu\n", __func__,
744 lo->plh_inode->i_ino);
745 inode = lo->plh_inode;
Christoph Hellwig7c5d1872014-09-10 08:23:29 -0700746
747 pnfs_layoutcommit_inode(inode, false);
748
Trond Myklebustfd9a8d72013-02-12 09:48:42 -0500749 spin_lock(&inode->i_lock);
750 list_del_init(&lo->plh_bulk_destroy);
Trond Myklebust9fd4b9f2016-02-22 17:46:34 -0500751 if (pnfs_mark_layout_stateid_invalid(lo, &lseg_list)) {
752 if (is_bulk_recall)
753 set_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
Trond Myklebustfd9a8d72013-02-12 09:48:42 -0500754 ret = -EAGAIN;
Trond Myklebust9fd4b9f2016-02-22 17:46:34 -0500755 }
Trond Myklebustfd9a8d72013-02-12 09:48:42 -0500756 spin_unlock(&inode->i_lock);
757 pnfs_free_lseg_list(&lseg_list);
Trond Myklebustb20135d2015-12-31 09:28:06 -0500758 /* Free all lsegs that are attached to commit buckets */
759 nfs_commit_inode(inode, 0);
Trond Myklebustfd9a8d72013-02-12 09:48:42 -0500760 pnfs_put_layout_hdr(lo);
761 iput(inode);
762 }
763 return ret;
764}
765
766int
767pnfs_destroy_layouts_byfsid(struct nfs_client *clp,
768 struct nfs_fsid *fsid,
769 bool is_recall)
770{
771 struct nfs_server *server;
772 LIST_HEAD(layout_list);
773
774 spin_lock(&clp->cl_lock);
775 rcu_read_lock();
776restart:
777 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
778 if (memcmp(&server->fsid, fsid, sizeof(*fsid)) != 0)
779 continue;
780 if (pnfs_layout_bulk_destroy_byserver_locked(clp,
781 server,
782 &layout_list) != 0)
783 goto restart;
784 }
785 rcu_read_unlock();
786 spin_unlock(&clp->cl_lock);
787
788 if (list_empty(&layout_list))
789 return 0;
790 return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
791}
792
793int
794pnfs_destroy_layouts_byclid(struct nfs_client *clp,
795 bool is_recall)
796{
797 struct nfs_server *server;
798 LIST_HEAD(layout_list);
799
800 spin_lock(&clp->cl_lock);
801 rcu_read_lock();
802restart:
803 list_for_each_entry_rcu(server, &clp->cl_superblocks, client_link) {
804 if (pnfs_layout_bulk_destroy_byserver_locked(clp,
805 server,
806 &layout_list) != 0)
807 goto restart;
808 }
809 rcu_read_unlock();
810 spin_unlock(&clp->cl_lock);
811
812 if (list_empty(&layout_list))
813 return 0;
814 return pnfs_layout_free_bulk_destroy_list(&layout_list, is_recall);
815}
816
Andy Adamson974cec82010-10-20 00:18:02 -0400817/*
818 * Called by the state manger to remove all layouts established under an
819 * expired lease.
820 */
821void
822pnfs_destroy_all_layouts(struct nfs_client *clp)
823{
Andy Adamsonc47abcf2011-06-15 17:52:40 -0400824 nfs4_deviceid_mark_client_invalid(clp);
825 nfs4_deviceid_purge_client(clp);
826
Trond Myklebustfd9a8d72013-02-12 09:48:42 -0500827 pnfs_destroy_layouts_byclid(clp, false);
Andy Adamson974cec82010-10-20 00:18:02 -0400828}
829
Fred Isamanfd6002e2011-01-06 11:36:22 +0000830/* update lo->plh_stateid with new if is more recent */
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000831void
832pnfs_set_layout_stateid(struct pnfs_layout_hdr *lo, const nfs4_stateid *new,
833 bool update_barrier)
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400834{
Trond Myklebustecebb802016-07-24 11:46:06 -0400835 u32 oldseq, newseq, new_barrier = 0;
Andy Adamson974cec82010-10-20 00:18:02 -0400836
Trond Myklebust2d2f24a2012-03-04 18:13:57 -0500837 oldseq = be32_to_cpu(lo->plh_stateid.seqid);
838 newseq = be32_to_cpu(new->seqid);
Trond Myklebust2a59a042016-09-03 11:20:04 -0400839
840 if (!pnfs_layout_is_valid(lo)) {
841 nfs4_stateid_copy(&lo->plh_stateid, new);
842 lo->plh_barrier = newseq;
843 pnfs_clear_layoutreturn_info(lo);
844 clear_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags);
845 return;
846 }
847 if (pnfs_seqid_is_newer(newseq, oldseq)) {
Trond Myklebustf597c532012-03-04 18:13:56 -0500848 nfs4_stateid_copy(&lo->plh_stateid, new);
Trond Myklebustecebb802016-07-24 11:46:06 -0400849 /*
850 * Because of wraparound, we want to keep the barrier
851 * "close" to the current seqids.
852 */
853 new_barrier = newseq - atomic_read(&lo->plh_outstanding);
Fred Isaman43f1b3d2011-01-06 11:36:30 +0000854 }
Trond Myklebustecebb802016-07-24 11:46:06 -0400855 if (update_barrier)
856 new_barrier = be32_to_cpu(new->seqid);
857 else if (new_barrier == 0)
858 return;
Trond Myklebust2a59a042016-09-03 11:20:04 -0400859 if (pnfs_seqid_is_newer(new_barrier, lo->plh_barrier))
Trond Myklebustecebb802016-07-24 11:46:06 -0400860 lo->plh_barrier = new_barrier;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400861}
862
Trond Myklebust19c54ab2012-10-05 16:56:58 -0700863static bool
864pnfs_layout_stateid_blocked(const struct pnfs_layout_hdr *lo,
865 const nfs4_stateid *stateid)
866{
867 u32 seqid = be32_to_cpu(stateid->seqid);
868
869 return !pnfs_seqid_is_newer(seqid, lo->plh_barrier);
870}
871
Fred Isamancf7d63f2011-01-06 11:36:25 +0000872/* lget is set to 1 if called from inside send_layoutget call chain */
873static bool
Trond Myklebuste1c06f82015-08-04 16:40:08 -0400874pnfs_layoutgets_blocked(const struct pnfs_layout_hdr *lo)
Fred Isamancf7d63f2011-01-06 11:36:25 +0000875{
Fred Isamanf7e89172011-01-06 11:36:32 +0000876 return lo->plh_block_lgets ||
Trond Myklebuste1c06f82015-08-04 16:40:08 -0400877 test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags);
Fred Isamancf7d63f2011-01-06 11:36:25 +0000878}
879
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400880/*
Jeff Layton183d9e72016-05-17 12:28:47 -0400881 * Get layout from server.
882 * for now, assume that whole file layouts are requested.
883 * arg->offset: 0
884 * arg->length: all ones
885 */
Benny Halevye5e94012010-10-20 00:18:01 -0400886static struct pnfs_layout_segment *
887send_layoutget(struct pnfs_layout_hdr *lo,
888 struct nfs_open_context *ctx,
Jeff Layton183d9e72016-05-17 12:28:47 -0400889 nfs4_stateid *stateid,
Trond Myklebuste144e532016-01-04 12:52:53 -0500890 const struct pnfs_layout_range *range,
Jeff Layton183d9e72016-05-17 12:28:47 -0400891 long *timeout, gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -0400892{
Fred Isamanb7edfaa2011-01-06 11:36:21 +0000893 struct inode *ino = lo->plh_inode;
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400894 struct nfs_server *server = NFS_SERVER(ino);
895 struct nfs4_layoutget *lgp;
Trond Myklebust2d89a1d2015-08-31 02:05:47 -0700896 loff_t i_size;
Benny Halevye5e94012010-10-20 00:18:01 -0400897
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400898 dprintk("--> %s\n", __func__);
899
Jeff Layton4f2e9dc2015-11-25 13:43:14 -0500900 /*
901 * Synchronously retrieve layout information from server and
902 * store in lseg. If we race with a concurrent seqid morphing
903 * op, then re-send the LAYOUTGET.
Andy Adamsonb1f69b72010-10-20 00:18:03 -0400904 */
Jeff Layton83026d82016-05-17 12:28:46 -0400905 lgp = kzalloc(sizeof(*lgp), gfp_flags);
906 if (lgp == NULL)
907 return ERR_PTR(-ENOMEM);
Jeff Layton4f2e9dc2015-11-25 13:43:14 -0500908
Jeff Layton83026d82016-05-17 12:28:46 -0400909 i_size = i_size_read(ino);
Jeff Layton4f2e9dc2015-11-25 13:43:14 -0500910
Jeff Layton83026d82016-05-17 12:28:46 -0400911 lgp->args.minlength = PAGE_SIZE;
912 if (lgp->args.minlength > range->length)
913 lgp->args.minlength = range->length;
914 if (range->iomode == IOMODE_READ) {
915 if (range->offset >= i_size)
916 lgp->args.minlength = 0;
917 else if (i_size - range->offset < lgp->args.minlength)
918 lgp->args.minlength = i_size - range->offset;
Jeff Laytond03ab292016-05-17 12:28:45 -0400919 }
Jeff Layton83026d82016-05-17 12:28:46 -0400920 lgp->args.maxcount = PNFS_LAYOUT_MAXSIZE;
921 pnfs_copy_range(&lgp->args.range, range);
922 lgp->args.type = server->pnfs_curr_ld->id;
923 lgp->args.inode = ino;
924 lgp->args.ctx = get_nfs_open_context(ctx);
Jeff Layton183d9e72016-05-17 12:28:47 -0400925 nfs4_stateid_copy(&lgp->args.stateid, stateid);
Jeff Layton83026d82016-05-17 12:28:46 -0400926 lgp->gfp_flags = gfp_flags;
927 lgp->cred = lo->plh_lc_cred;
Weston Andros Adamson35124a02011-03-24 16:48:21 -0400928
Jeff Layton183d9e72016-05-17 12:28:47 -0400929 return nfs4_proc_layoutget(lgp, timeout, gfp_flags);
Andy Adamson974cec82010-10-20 00:18:02 -0400930}
931
Trond Myklebust24956802013-03-20 13:03:00 -0400932static void pnfs_clear_layoutcommit(struct inode *inode,
933 struct list_head *head)
934{
935 struct nfs_inode *nfsi = NFS_I(inode);
936 struct pnfs_layout_segment *lseg, *tmp;
937
938 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
939 return;
940 list_for_each_entry_safe(lseg, tmp, &nfsi->layout->plh_segs, pls_list) {
941 if (!test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
942 continue;
943 pnfs_lseg_dec_and_remove_zero(lseg, head);
944 }
945}
946
Tom Haynesd67ae822014-12-11 17:02:04 -0500947void pnfs_clear_layoutreturn_waitbit(struct pnfs_layout_hdr *lo)
948{
949 clear_bit_unlock(NFS_LAYOUT_RETURN, &lo->plh_flags);
Trond Myklebust24b049f2016-10-17 17:54:32 -0400950 clear_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags);
Tom Haynesd67ae822014-12-11 17:02:04 -0500951 smp_mb__after_atomic();
952 wake_up_bit(&lo->plh_flags, NFS_LAYOUT_RETURN);
Trond Myklebust7f273922015-07-09 18:40:01 +0200953 rpc_wake_up(&NFS_SERVER(lo->plh_inode)->roc_rpcwaitq);
Tom Haynesd67ae822014-12-11 17:02:04 -0500954}
955
Trond Myklebust13c13a62016-01-26 23:12:11 -0500956static bool
Trond Myklebuste5fd1902016-07-21 12:44:15 -0400957pnfs_prepare_layoutreturn(struct pnfs_layout_hdr *lo,
958 nfs4_stateid *stateid,
959 enum pnfs_iomode *iomode)
Trond Myklebust13c13a62016-01-26 23:12:11 -0500960{
Trond Myklebustbf0291d2016-09-03 10:39:51 -0400961 /* Serialise LAYOUTGET/LAYOUTRETURN */
962 if (atomic_read(&lo->plh_outstanding) != 0)
963 return false;
Trond Myklebust24b049f2016-10-17 17:54:32 -0400964 if (test_and_set_bit(NFS_LAYOUT_RETURN_LOCK, &lo->plh_flags))
Trond Myklebust13c13a62016-01-26 23:12:11 -0500965 return false;
Trond Myklebust24b049f2016-10-17 17:54:32 -0400966 set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags);
Trond Myklebust13c13a62016-01-26 23:12:11 -0500967 pnfs_get_layout_hdr(lo);
Trond Myklebuste5fd1902016-07-21 12:44:15 -0400968 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags)) {
969 if (stateid != NULL) {
970 nfs4_stateid_copy(stateid, &lo->plh_stateid);
971 if (lo->plh_return_seq != 0)
972 stateid->seqid = cpu_to_be32(lo->plh_return_seq);
973 }
974 if (iomode != NULL)
975 *iomode = lo->plh_return_iomode;
976 pnfs_clear_layoutreturn_info(lo);
977 return true;
978 }
979 if (stateid != NULL)
980 nfs4_stateid_copy(stateid, &lo->plh_stateid);
981 if (iomode != NULL)
982 *iomode = IOMODE_ANY;
Trond Myklebust13c13a62016-01-26 23:12:11 -0500983 return true;
984}
985
Peng Taof40eb5d2014-09-06 00:53:22 +0800986static int
Trond Myklebusted429d62016-01-04 12:30:55 -0500987pnfs_send_layoutreturn(struct pnfs_layout_hdr *lo, const nfs4_stateid *stateid,
Peng Tao6c166052014-11-17 09:30:40 +0800988 enum pnfs_iomode iomode, bool sync)
Peng Taof40eb5d2014-09-06 00:53:22 +0800989{
990 struct inode *ino = lo->plh_inode;
991 struct nfs4_layoutreturn *lrp;
992 int status = 0;
993
Trond Myklebuste4af4402015-02-05 17:05:08 -0500994 lrp = kzalloc(sizeof(*lrp), GFP_NOFS);
Peng Taof40eb5d2014-09-06 00:53:22 +0800995 if (unlikely(lrp == NULL)) {
996 status = -ENOMEM;
997 spin_lock(&ino->i_lock);
Tom Haynesd67ae822014-12-11 17:02:04 -0500998 pnfs_clear_layoutreturn_waitbit(lo);
Peng Taof40eb5d2014-09-06 00:53:22 +0800999 spin_unlock(&ino->i_lock);
1000 pnfs_put_layout_hdr(lo);
1001 goto out;
1002 }
1003
Trond Myklebusted429d62016-01-04 12:30:55 -05001004 nfs4_stateid_copy(&lrp->args.stateid, stateid);
Peng Taof40eb5d2014-09-06 00:53:22 +08001005 lrp->args.layout_type = NFS_SERVER(ino)->pnfs_curr_ld->id;
1006 lrp->args.inode = ino;
Peng Tao15eb67c2014-11-17 09:30:36 +08001007 lrp->args.range.iomode = iomode;
1008 lrp->args.range.offset = 0;
1009 lrp->args.range.length = NFS4_MAX_UINT64;
Peng Taof40eb5d2014-09-06 00:53:22 +08001010 lrp->args.layout = lo;
1011 lrp->clp = NFS_SERVER(ino)->nfs_client;
1012 lrp->cred = lo->plh_lc_cred;
1013
Peng Tao6c166052014-11-17 09:30:40 +08001014 status = nfs4_proc_layoutreturn(lrp, sync);
Peng Taof40eb5d2014-09-06 00:53:22 +08001015out:
1016 dprintk("<-- %s status: %d\n", __func__, status);
1017 return status;
1018}
1019
Trond Myklebust13c13a62016-01-26 23:12:11 -05001020/* Return true if layoutreturn is needed */
1021static bool
1022pnfs_layout_need_return(struct pnfs_layout_hdr *lo)
1023{
1024 struct pnfs_layout_segment *s;
1025
Trond Myklebust2370abd2016-01-27 20:32:50 -05001026 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
Trond Myklebust13c13a62016-01-26 23:12:11 -05001027 return false;
1028
1029 /* Defer layoutreturn until all lsegs are done */
1030 list_for_each_entry(s, &lo->plh_segs, pls_list) {
1031 if (test_bit(NFS_LSEG_LAYOUTRETURN, &s->pls_flags))
1032 return false;
1033 }
1034
1035 return true;
1036}
1037
1038static void pnfs_layoutreturn_before_put_layout_hdr(struct pnfs_layout_hdr *lo)
1039{
1040 struct inode *inode= lo->plh_inode;
1041
Trond Myklebust2370abd2016-01-27 20:32:50 -05001042 if (!test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
Trond Myklebust13c13a62016-01-26 23:12:11 -05001043 return;
1044 spin_lock(&inode->i_lock);
1045 if (pnfs_layout_need_return(lo)) {
1046 nfs4_stateid stateid;
1047 enum pnfs_iomode iomode;
1048 bool send;
1049
Trond Myklebuste5fd1902016-07-21 12:44:15 -04001050 send = pnfs_prepare_layoutreturn(lo, &stateid, &iomode);
Trond Myklebust13c13a62016-01-26 23:12:11 -05001051 spin_unlock(&inode->i_lock);
1052 if (send) {
1053 /* Send an async layoutreturn so we dont deadlock */
1054 pnfs_send_layoutreturn(lo, &stateid, iomode, false);
1055 }
1056 } else
1057 spin_unlock(&inode->i_lock);
1058}
1059
Andy Adamson293b3b02012-06-20 15:03:34 -04001060/*
1061 * Initiates a LAYOUTRETURN(FILE), and removes the pnfs_layout_hdr
1062 * when the layout segment list is empty.
1063 *
1064 * Note that a pnfs_layout_hdr can exist with an empty layout segment
1065 * list when LAYOUTGET has failed, or when LAYOUTGET succeeded, but the
1066 * deviceid is marked invalid.
1067 */
Benny Halevycbe82602011-05-22 19:52:37 +03001068int
1069_pnfs_return_layout(struct inode *ino)
1070{
1071 struct pnfs_layout_hdr *lo = NULL;
1072 struct nfs_inode *nfsi = NFS_I(ino);
1073 LIST_HEAD(tmp_list);
Benny Halevycbe82602011-05-22 19:52:37 +03001074 nfs4_stateid stateid;
Andy Adamson293b3b02012-06-20 15:03:34 -04001075 int status = 0, empty;
Trond Myklebust7f273922015-07-09 18:40:01 +02001076 bool send;
Benny Halevycbe82602011-05-22 19:52:37 +03001077
Andy Adamson366d5052012-06-20 15:03:33 -04001078 dprintk("NFS: %s for inode %lu\n", __func__, ino->i_ino);
Benny Halevycbe82602011-05-22 19:52:37 +03001079
1080 spin_lock(&ino->i_lock);
1081 lo = nfsi->layout;
Trond Myklebuste5929f32012-09-21 16:37:02 -04001082 if (!lo) {
Benny Halevycbe82602011-05-22 19:52:37 +03001083 spin_unlock(&ino->i_lock);
Andy Adamson293b3b02012-06-20 15:03:34 -04001084 dprintk("NFS: %s no layout to return\n", __func__);
1085 goto out;
Benny Halevycbe82602011-05-22 19:52:37 +03001086 }
Benny Halevycbe82602011-05-22 19:52:37 +03001087 /* Reference matched in nfs4_layoutreturn_release */
Trond Myklebust70c3bd22012-09-18 20:51:13 -04001088 pnfs_get_layout_hdr(lo);
Andy Adamson293b3b02012-06-20 15:03:34 -04001089 empty = list_empty(&lo->plh_segs);
Trond Myklebust24956802013-03-20 13:03:00 -04001090 pnfs_clear_layoutcommit(ino, &tmp_list);
Jeff Layton6d597e12016-05-17 12:28:42 -04001091 pnfs_mark_matching_lsegs_invalid(lo, &tmp_list, NULL, 0);
Christoph Hellwigc88953d2014-09-10 08:23:31 -07001092
1093 if (NFS_SERVER(ino)->pnfs_curr_ld->return_range) {
1094 struct pnfs_layout_range range = {
1095 .iomode = IOMODE_ANY,
1096 .offset = 0,
1097 .length = NFS4_MAX_UINT64,
1098 };
1099 NFS_SERVER(ino)->pnfs_curr_ld->return_range(lo, &range);
1100 }
1101
Andy Adamson293b3b02012-06-20 15:03:34 -04001102 /* Don't send a LAYOUTRETURN if list was initially empty */
1103 if (empty) {
1104 spin_unlock(&ino->i_lock);
Andy Adamson293b3b02012-06-20 15:03:34 -04001105 dprintk("NFS: %s no layout segments to return\n", __func__);
Trond Myklebust7f273922015-07-09 18:40:01 +02001106 goto out_put_layout_hdr;
Andy Adamson293b3b02012-06-20 15:03:34 -04001107 }
Christoph Hellwig47abade2014-08-21 11:09:22 -05001108
Trond Myklebuste5fd1902016-07-21 12:44:15 -04001109 send = pnfs_prepare_layoutreturn(lo, &stateid, NULL);
Benny Halevycbe82602011-05-22 19:52:37 +03001110 spin_unlock(&ino->i_lock);
1111 pnfs_free_lseg_list(&tmp_list);
Trond Myklebust7f273922015-07-09 18:40:01 +02001112 if (send)
Trond Myklebusted429d62016-01-04 12:30:55 -05001113 status = pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true);
Trond Myklebust7f273922015-07-09 18:40:01 +02001114out_put_layout_hdr:
1115 pnfs_put_layout_hdr(lo);
Benny Halevycbe82602011-05-22 19:52:37 +03001116out:
1117 dprintk("<-- %s status: %d\n", __func__, status);
1118 return status;
1119}
Andy Adamson0a57cda2012-04-27 17:53:50 -04001120EXPORT_SYMBOL_GPL(_pnfs_return_layout);
Benny Halevycbe82602011-05-22 19:52:37 +03001121
Trond Myklebust24028672013-03-20 13:23:33 -04001122int
1123pnfs_commit_and_return_layout(struct inode *inode)
1124{
1125 struct pnfs_layout_hdr *lo;
1126 int ret;
1127
1128 spin_lock(&inode->i_lock);
1129 lo = NFS_I(inode)->layout;
1130 if (lo == NULL) {
1131 spin_unlock(&inode->i_lock);
1132 return 0;
1133 }
1134 pnfs_get_layout_hdr(lo);
1135 /* Block new layoutgets and read/write to ds */
1136 lo->plh_block_lgets++;
1137 spin_unlock(&inode->i_lock);
1138 filemap_fdatawait(inode->i_mapping);
1139 ret = pnfs_layoutcommit_inode(inode, true);
1140 if (ret == 0)
1141 ret = _pnfs_return_layout(inode);
1142 spin_lock(&inode->i_lock);
1143 lo->plh_block_lgets--;
1144 spin_unlock(&inode->i_lock);
1145 pnfs_put_layout_hdr(lo);
1146 return ret;
1147}
1148
Fred Isamanf7e89172011-01-06 11:36:32 +00001149bool pnfs_roc(struct inode *ino)
1150{
Trond Myklebust40dd4b72015-01-24 13:54:37 -05001151 struct nfs_inode *nfsi = NFS_I(ino);
1152 struct nfs_open_context *ctx;
1153 struct nfs4_state *state;
Fred Isamanf7e89172011-01-06 11:36:32 +00001154 struct pnfs_layout_hdr *lo;
1155 struct pnfs_layout_segment *lseg, *tmp;
Peng Tao193e3aa2014-11-17 09:30:41 +08001156 nfs4_stateid stateid;
Fred Isamanf7e89172011-01-06 11:36:32 +00001157 LIST_HEAD(tmp_list);
Peng Taoe755d632015-08-19 13:49:19 +08001158 bool found = false, layoutreturn = false, roc = false;
Fred Isamanf7e89172011-01-06 11:36:32 +00001159
1160 spin_lock(&ino->i_lock);
Trond Myklebust40dd4b72015-01-24 13:54:37 -05001161 lo = nfsi->layout;
Peng Tao39761432015-08-21 12:49:44 +08001162 if (!lo || test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags))
Trond Myklebust40dd4b72015-01-24 13:54:37 -05001163 goto out_noroc;
1164
Peng Taoe755d632015-08-19 13:49:19 +08001165 /* no roc if we hold a delegation */
Trond Myklebust40dd4b72015-01-24 13:54:37 -05001166 if (nfs4_check_delegation(ino, FMODE_READ))
1167 goto out_noroc;
1168
1169 list_for_each_entry(ctx, &nfsi->open_files, list) {
1170 state = ctx->state;
1171 /* Don't return layout if there is open file state */
1172 if (state != NULL && state->state != 0)
1173 goto out_noroc;
1174 }
1175
Peng Taoe755d632015-08-19 13:49:19 +08001176 /* always send layoutreturn if being marked so */
Trond Myklebuste5fd1902016-07-21 12:44:15 -04001177 if (test_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags))
1178 layoutreturn = pnfs_prepare_layoutreturn(lo,
1179 &stateid, NULL);
Peng Taoe755d632015-08-19 13:49:19 +08001180
Fred Isamanf7e89172011-01-06 11:36:32 +00001181 list_for_each_entry_safe(lseg, tmp, &lo->plh_segs, pls_list)
Peng Taoe755d632015-08-19 13:49:19 +08001182 /* If we are sending layoutreturn, invalidate all valid lsegs */
1183 if (layoutreturn || test_bit(NFS_LSEG_ROC, &lseg->pls_flags)) {
Fred Isamanf7e89172011-01-06 11:36:32 +00001184 mark_lseg_invalid(lseg, &tmp_list);
1185 found = true;
1186 }
Peng Tao500d7012015-09-22 11:35:22 +08001187 /* ROC in two conditions:
Peng Taoe755d632015-08-19 13:49:19 +08001188 * 1. there are ROC lsegs
1189 * 2. we don't send layoutreturn
Peng Taoe755d632015-08-19 13:49:19 +08001190 */
Peng Tao500d7012015-09-22 11:35:22 +08001191 if (found && !layoutreturn) {
1192 /* lo ref dropped in pnfs_roc_release() */
1193 pnfs_get_layout_hdr(lo);
Peng Taoe755d632015-08-19 13:49:19 +08001194 roc = true;
Peng Tao500d7012015-09-22 11:35:22 +08001195 }
Peng Taoe755d632015-08-19 13:49:19 +08001196
1197out_noroc:
Fred Isamanf7e89172011-01-06 11:36:32 +00001198 spin_unlock(&ino->i_lock);
1199 pnfs_free_lseg_list(&tmp_list);
Trond Myklebust71401712015-03-25 12:36:13 -04001200 pnfs_layoutcommit_inode(ino, true);
Peng Taoe755d632015-08-19 13:49:19 +08001201 if (layoutreturn)
Trond Myklebusted429d62016-01-04 12:30:55 -05001202 pnfs_send_layoutreturn(lo, &stateid, IOMODE_ANY, true);
Peng Taoe755d632015-08-19 13:49:19 +08001203 return roc;
Fred Isamanf7e89172011-01-06 11:36:32 +00001204}
1205
1206void pnfs_roc_release(struct inode *ino)
1207{
1208 struct pnfs_layout_hdr *lo;
1209
1210 spin_lock(&ino->i_lock);
1211 lo = NFS_I(ino)->layout;
Trond Myklebust5c4a79f2015-08-04 15:57:13 -04001212 pnfs_clear_layoutreturn_waitbit(lo);
Trond Myklebust6622c3e2012-09-20 17:23:11 -04001213 if (atomic_dec_and_test(&lo->plh_refcount)) {
1214 pnfs_detach_layout_hdr(lo);
1215 spin_unlock(&ino->i_lock);
1216 pnfs_free_layout_hdr(lo);
1217 } else
1218 spin_unlock(&ino->i_lock);
Fred Isamanf7e89172011-01-06 11:36:32 +00001219}
1220
1221void pnfs_roc_set_barrier(struct inode *ino, u32 barrier)
1222{
1223 struct pnfs_layout_hdr *lo;
1224
1225 spin_lock(&ino->i_lock);
1226 lo = NFS_I(ino)->layout;
Trond Myklebust0f35ad62012-10-04 16:28:17 -07001227 if (pnfs_seqid_is_newer(barrier, lo->plh_barrier))
Fred Isamanf7e89172011-01-06 11:36:32 +00001228 lo->plh_barrier = barrier;
1229 spin_unlock(&ino->i_lock);
Trond Myklebust6a463beb2015-08-20 15:40:47 -05001230 trace_nfs4_layoutreturn_on_close(ino, 0);
Fred Isamanf7e89172011-01-06 11:36:32 +00001231}
1232
Trond Myklebust4ff376f2015-08-18 23:23:21 -05001233void pnfs_roc_get_barrier(struct inode *ino, u32 *barrier)
Fred Isamanf7e89172011-01-06 11:36:32 +00001234{
1235 struct nfs_inode *nfsi = NFS_I(ino);
Trond Myklebust7fdab062012-09-20 20:15:57 -04001236 struct pnfs_layout_hdr *lo;
Trond Myklebust7fdab062012-09-20 20:15:57 -04001237 u32 current_seqid;
Fred Isamanf7e89172011-01-06 11:36:32 +00001238
1239 spin_lock(&ino->i_lock);
Trond Myklebust7fdab062012-09-20 20:15:57 -04001240 lo = nfsi->layout;
1241 current_seqid = be32_to_cpu(lo->plh_stateid.seqid);
Fred Isamanf7e89172011-01-06 11:36:32 +00001242
Trond Myklebust7fdab062012-09-20 20:15:57 -04001243 /* Since close does not return a layout stateid for use as
1244 * a barrier, we choose the worst-case barrier.
1245 */
1246 *barrier = current_seqid + atomic_read(&lo->plh_outstanding);
Fred Isamanf7e89172011-01-06 11:36:32 +00001247 spin_unlock(&ino->i_lock);
Fred Isamanf7e89172011-01-06 11:36:32 +00001248}
1249
Peng Tao500d7012015-09-22 11:35:22 +08001250bool pnfs_wait_on_layoutreturn(struct inode *ino, struct rpc_task *task)
1251{
1252 struct nfs_inode *nfsi = NFS_I(ino);
1253 struct pnfs_layout_hdr *lo;
1254 bool sleep = false;
1255
1256 /* we might not have grabbed lo reference. so need to check under
1257 * i_lock */
1258 spin_lock(&ino->i_lock);
1259 lo = nfsi->layout;
Trond Myklebust4c4d4be2016-11-18 15:21:30 -05001260 if (lo && test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
Peng Tao500d7012015-09-22 11:35:22 +08001261 rpc_sleep_on(&NFS_SERVER(ino)->roc_rpcwaitq, task, NULL);
Trond Myklebust4c4d4be2016-11-18 15:21:30 -05001262 sleep = true;
1263 }
1264 spin_unlock(&ino->i_lock);
Peng Tao500d7012015-09-22 11:35:22 +08001265 return sleep;
1266}
1267
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001268/*
1269 * Compare two layout segments for sorting into layout cache.
1270 * We want to preferentially return RW over RO layouts, so ensure those
1271 * are seen first.
1272 */
1273static s64
Trond Myklebust7dc0ac72013-06-03 11:30:24 -04001274pnfs_lseg_range_cmp(const struct pnfs_layout_range *l1,
Trond Myklebust3cb2df12013-06-03 11:24:36 -04001275 const struct pnfs_layout_range *l2)
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001276{
Benny Halevyfb3296e2011-05-22 19:47:26 +03001277 s64 d;
1278
1279 /* high offset > low offset */
1280 d = l1->offset - l2->offset;
1281 if (d)
1282 return d;
1283
1284 /* short length > long length */
1285 d = l2->length - l1->length;
1286 if (d)
1287 return d;
1288
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001289 /* read > read/write */
Benny Halevyfb3296e2011-05-22 19:47:26 +03001290 return (int)(l1->iomode == IOMODE_READ) - (int)(l2->iomode == IOMODE_READ);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001291}
1292
Trond Myklebust03772d22015-08-25 08:54:17 -04001293static bool
1294pnfs_lseg_range_is_after(const struct pnfs_layout_range *l1,
1295 const struct pnfs_layout_range *l2)
Andy Adamson974cec82010-10-20 00:18:02 -04001296{
Trond Myklebust03772d22015-08-25 08:54:17 -04001297 return pnfs_lseg_range_cmp(l1, l2) > 0;
1298}
1299
1300static bool
1301pnfs_lseg_no_merge(struct pnfs_layout_segment *lseg,
1302 struct pnfs_layout_segment *old)
1303{
1304 return false;
1305}
1306
1307void
1308pnfs_generic_layout_insert_lseg(struct pnfs_layout_hdr *lo,
1309 struct pnfs_layout_segment *lseg,
1310 bool (*is_after)(const struct pnfs_layout_range *,
1311 const struct pnfs_layout_range *),
1312 bool (*do_merge)(struct pnfs_layout_segment *,
1313 struct pnfs_layout_segment *),
1314 struct list_head *free_me)
1315{
1316 struct pnfs_layout_segment *lp, *tmp;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001317
Andy Adamson974cec82010-10-20 00:18:02 -04001318 dprintk("%s:Begin\n", __func__);
1319
Trond Myklebust03772d22015-08-25 08:54:17 -04001320 list_for_each_entry_safe(lp, tmp, &lo->plh_segs, pls_list) {
1321 if (test_bit(NFS_LSEG_VALID, &lp->pls_flags) == 0)
1322 continue;
1323 if (do_merge(lseg, lp)) {
1324 mark_lseg_invalid(lp, free_me);
1325 continue;
1326 }
1327 if (is_after(&lseg->pls_range, &lp->pls_range))
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001328 continue;
Fred Isaman566052c2011-01-06 11:36:20 +00001329 list_add_tail(&lseg->pls_list, &lp->pls_list);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001330 dprintk("%s: inserted lseg %p "
1331 "iomode %d offset %llu length %llu before "
1332 "lp %p iomode %d offset %llu length %llu\n",
Fred Isaman566052c2011-01-06 11:36:20 +00001333 __func__, lseg, lseg->pls_range.iomode,
1334 lseg->pls_range.offset, lseg->pls_range.length,
1335 lp, lp->pls_range.iomode, lp->pls_range.offset,
1336 lp->pls_range.length);
Benny Halevyfb3296e2011-05-22 19:47:26 +03001337 goto out;
Andy Adamson974cec82010-10-20 00:18:02 -04001338 }
Benny Halevyfb3296e2011-05-22 19:47:26 +03001339 list_add_tail(&lseg->pls_list, &lo->plh_segs);
1340 dprintk("%s: inserted lseg %p "
1341 "iomode %d offset %llu length %llu at tail\n",
1342 __func__, lseg, lseg->pls_range.iomode,
1343 lseg->pls_range.offset, lseg->pls_range.length);
1344out:
Trond Myklebust70c3bd22012-09-18 20:51:13 -04001345 pnfs_get_layout_hdr(lo);
Andy Adamson974cec82010-10-20 00:18:02 -04001346
1347 dprintk("%s:Return\n", __func__);
Benny Halevye5e94012010-10-20 00:18:01 -04001348}
Trond Myklebust03772d22015-08-25 08:54:17 -04001349EXPORT_SYMBOL_GPL(pnfs_generic_layout_insert_lseg);
1350
1351static void
1352pnfs_layout_insert_lseg(struct pnfs_layout_hdr *lo,
1353 struct pnfs_layout_segment *lseg,
1354 struct list_head *free_me)
1355{
1356 struct inode *inode = lo->plh_inode;
1357 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
1358
1359 if (ld->add_lseg != NULL)
1360 ld->add_lseg(lo, lseg, free_me);
1361 else
1362 pnfs_generic_layout_insert_lseg(lo, lseg,
1363 pnfs_lseg_range_is_after,
1364 pnfs_lseg_no_merge,
1365 free_me);
1366}
Benny Halevye5e94012010-10-20 00:18:01 -04001367
1368static struct pnfs_layout_hdr *
Peng Tao9fa40752011-07-30 20:52:32 -04001369alloc_init_layout_hdr(struct inode *ino,
1370 struct nfs_open_context *ctx,
1371 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -04001372{
1373 struct pnfs_layout_hdr *lo;
1374
Benny Halevy636fb9c2011-05-22 19:51:33 +03001375 lo = pnfs_alloc_layout_hdr(ino, gfp_flags);
Benny Halevye5e94012010-10-20 00:18:01 -04001376 if (!lo)
1377 return NULL;
Fred Isamancc6e5342011-01-06 11:36:28 +00001378 atomic_set(&lo->plh_refcount, 1);
Fred Isamanb7edfaa2011-01-06 11:36:21 +00001379 INIT_LIST_HEAD(&lo->plh_layouts);
1380 INIT_LIST_HEAD(&lo->plh_segs);
Trond Myklebustfd9a8d72013-02-12 09:48:42 -05001381 INIT_LIST_HEAD(&lo->plh_bulk_destroy);
Fred Isamanb7edfaa2011-01-06 11:36:21 +00001382 lo->plh_inode = ino;
Trond Myklebust5cc22162013-05-21 09:26:49 -04001383 lo->plh_lc_cred = get_rpccred(ctx->cred);
Trond Myklebust67a3b722016-06-17 16:48:19 -04001384 lo->plh_flags |= 1 << NFS_LAYOUT_INVALID_STID;
Benny Halevye5e94012010-10-20 00:18:01 -04001385 return lo;
1386}
1387
1388static struct pnfs_layout_hdr *
Peng Tao9fa40752011-07-30 20:52:32 -04001389pnfs_find_alloc_layout(struct inode *ino,
1390 struct nfs_open_context *ctx,
1391 gfp_t gfp_flags)
Trond Myklebuste5241e42016-06-17 16:48:20 -04001392 __releases(&ino->i_lock)
1393 __acquires(&ino->i_lock)
Benny Halevye5e94012010-10-20 00:18:01 -04001394{
1395 struct nfs_inode *nfsi = NFS_I(ino);
1396 struct pnfs_layout_hdr *new = NULL;
1397
1398 dprintk("%s Begin ino=%p layout=%p\n", __func__, ino, nfsi->layout);
1399
Trond Myklebust251ec412012-10-02 15:41:05 -07001400 if (nfsi->layout != NULL)
1401 goto out_existing;
Benny Halevye5e94012010-10-20 00:18:01 -04001402 spin_unlock(&ino->i_lock);
Peng Tao9fa40752011-07-30 20:52:32 -04001403 new = alloc_init_layout_hdr(ino, ctx, gfp_flags);
Benny Halevye5e94012010-10-20 00:18:01 -04001404 spin_lock(&ino->i_lock);
1405
Trond Myklebust251ec412012-10-02 15:41:05 -07001406 if (likely(nfsi->layout == NULL)) { /* Won the race? */
Benny Halevye5e94012010-10-20 00:18:01 -04001407 nfsi->layout = new;
Trond Myklebust251ec412012-10-02 15:41:05 -07001408 return new;
Yanchuan Nian7175fe92012-10-31 16:05:48 +08001409 } else if (new != NULL)
1410 pnfs_free_layout_hdr(new);
Trond Myklebust251ec412012-10-02 15:41:05 -07001411out_existing:
1412 pnfs_get_layout_hdr(nfsi->layout);
Benny Halevye5e94012010-10-20 00:18:01 -04001413 return nfsi->layout;
1414}
1415
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001416/*
1417 * iomode matching rules:
Tom Haynesc7d73af2016-05-25 07:31:14 -07001418 * iomode lseg strict match
1419 * iomode
1420 * ----- ----- ------ -----
1421 * ANY READ N/A true
1422 * ANY RW N/A true
1423 * RW READ N/A false
1424 * RW RW N/A true
1425 * READ READ N/A true
1426 * READ RW true false
1427 * READ RW false true
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001428 */
Trond Myklebust3cb2df12013-06-03 11:24:36 -04001429static bool
Trond Myklebust7dc0ac72013-06-03 11:30:24 -04001430pnfs_lseg_range_match(const struct pnfs_layout_range *ls_range,
Tom Haynesc7d73af2016-05-25 07:31:14 -07001431 const struct pnfs_layout_range *range,
1432 bool strict_iomode)
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001433{
Benny Halevyfb3296e2011-05-22 19:47:26 +03001434 struct pnfs_layout_range range1;
1435
1436 if ((range->iomode == IOMODE_RW &&
1437 ls_range->iomode != IOMODE_RW) ||
Tom Haynesc7d73af2016-05-25 07:31:14 -07001438 (range->iomode != ls_range->iomode &&
1439 strict_iomode == true) ||
Trond Myklebust7dc0ac72013-06-03 11:30:24 -04001440 !pnfs_lseg_range_intersecting(ls_range, range))
Benny Halevyfb3296e2011-05-22 19:47:26 +03001441 return 0;
1442
1443 /* range1 covers only the first byte in the range */
1444 range1 = *range;
1445 range1.length = 1;
Trond Myklebust7dc0ac72013-06-03 11:30:24 -04001446 return pnfs_lseg_range_contained(ls_range, &range1);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001447}
1448
1449/*
1450 * lookup range in layout
1451 */
Benny Halevye5e94012010-10-20 00:18:01 -04001452static struct pnfs_layout_segment *
Benny Halevyfb3296e2011-05-22 19:47:26 +03001453pnfs_find_lseg(struct pnfs_layout_hdr *lo,
Tom Haynesc7d73af2016-05-25 07:31:14 -07001454 struct pnfs_layout_range *range,
1455 bool strict_iomode)
Benny Halevye5e94012010-10-20 00:18:01 -04001456{
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001457 struct pnfs_layout_segment *lseg, *ret = NULL;
1458
1459 dprintk("%s:Begin\n", __func__);
1460
Fred Isamanb7edfaa2011-01-06 11:36:21 +00001461 list_for_each_entry(lseg, &lo->plh_segs, pls_list) {
Fred Isaman4541d162011-01-06 11:36:23 +00001462 if (test_bit(NFS_LSEG_VALID, &lseg->pls_flags) &&
Peng Taoce6ab4f2014-09-06 00:53:24 +08001463 !test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags) &&
Tom Haynesc7d73af2016-05-25 07:31:14 -07001464 pnfs_lseg_range_match(&lseg->pls_range, range,
1465 strict_iomode)) {
Trond Myklebust9369a432012-09-18 20:57:08 -04001466 ret = pnfs_get_lseg(lseg);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001467 break;
1468 }
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001469 }
1470
1471 dprintk("%s:Return lseg %p ref %d\n",
Fred Isaman4541d162011-01-06 11:36:23 +00001472 __func__, ret, ret ? atomic_read(&ret->pls_refcount) : 0);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001473 return ret;
Benny Halevye5e94012010-10-20 00:18:01 -04001474}
1475
1476/*
Andy Adamsond23d61c2012-05-23 05:02:37 -04001477 * Use mdsthreshold hints set at each OPEN to determine if I/O should go
1478 * to the MDS or over pNFS
1479 *
1480 * The nfs_inode read_io and write_io fields are cumulative counters reset
1481 * when there are no layout segments. Note that in pnfs_update_layout iomode
1482 * is set to IOMODE_READ for a READ request, and set to IOMODE_RW for a
1483 * WRITE request.
1484 *
1485 * A return of true means use MDS I/O.
1486 *
1487 * From rfc 5661:
1488 * If a file's size is smaller than the file size threshold, data accesses
1489 * SHOULD be sent to the metadata server. If an I/O request has a length that
1490 * is below the I/O size threshold, the I/O SHOULD be sent to the metadata
1491 * server. If both file size and I/O size are provided, the client SHOULD
1492 * reach or exceed both thresholds before sending its read or write
1493 * requests to the data server.
1494 */
1495static bool pnfs_within_mdsthreshold(struct nfs_open_context *ctx,
1496 struct inode *ino, int iomode)
1497{
1498 struct nfs4_threshold *t = ctx->mdsthreshold;
1499 struct nfs_inode *nfsi = NFS_I(ino);
1500 loff_t fsize = i_size_read(ino);
1501 bool size = false, size_set = false, io = false, io_set = false, ret = false;
1502
1503 if (t == NULL)
1504 return ret;
1505
1506 dprintk("%s bm=0x%x rd_sz=%llu wr_sz=%llu rd_io=%llu wr_io=%llu\n",
1507 __func__, t->bm, t->rd_sz, t->wr_sz, t->rd_io_sz, t->wr_io_sz);
1508
1509 switch (iomode) {
1510 case IOMODE_READ:
1511 if (t->bm & THRESHOLD_RD) {
1512 dprintk("%s fsize %llu\n", __func__, fsize);
1513 size_set = true;
1514 if (fsize < t->rd_sz)
1515 size = true;
1516 }
1517 if (t->bm & THRESHOLD_RD_IO) {
1518 dprintk("%s nfsi->read_io %llu\n", __func__,
1519 nfsi->read_io);
1520 io_set = true;
1521 if (nfsi->read_io < t->rd_io_sz)
1522 io = true;
1523 }
1524 break;
1525 case IOMODE_RW:
1526 if (t->bm & THRESHOLD_WR) {
1527 dprintk("%s fsize %llu\n", __func__, fsize);
1528 size_set = true;
1529 if (fsize < t->wr_sz)
1530 size = true;
1531 }
1532 if (t->bm & THRESHOLD_WR_IO) {
1533 dprintk("%s nfsi->write_io %llu\n", __func__,
1534 nfsi->write_io);
1535 io_set = true;
1536 if (nfsi->write_io < t->wr_io_sz)
1537 io = true;
1538 }
1539 break;
1540 }
1541 if (size_set && io_set) {
1542 if (size && io)
1543 ret = true;
1544 } else if (size || io)
1545 ret = true;
1546
1547 dprintk("<-- %s size %d io %d ret %d\n", __func__, size, io, ret);
1548 return ret;
1549}
1550
Peng Taoaa8a45e2014-12-01 08:22:23 +08001551static bool pnfs_prepare_to_retry_layoutget(struct pnfs_layout_hdr *lo)
1552{
1553 /*
1554 * send layoutcommit as it can hold up layoutreturn due to lseg
1555 * reference
1556 */
1557 pnfs_layoutcommit_inode(lo->plh_inode, false);
1558 return !wait_on_bit_action(&lo->plh_flags, NFS_LAYOUT_RETURN,
Trond Myklebust2e5b29f2015-12-14 16:25:11 -05001559 nfs_wait_bit_killable,
Peng Taoaa8a45e2014-12-01 08:22:23 +08001560 TASK_UNINTERRUPTIBLE);
1561}
1562
Tom Haynesd67ae822014-12-11 17:02:04 -05001563static void pnfs_clear_first_layoutget(struct pnfs_layout_hdr *lo)
1564{
1565 unsigned long *bitlock = &lo->plh_flags;
1566
1567 clear_bit_unlock(NFS_LAYOUT_FIRST_LAYOUTGET, bitlock);
1568 smp_mb__after_atomic();
1569 wake_up_bit(bitlock, NFS_LAYOUT_FIRST_LAYOUTGET);
1570}
1571
Andy Adamsond23d61c2012-05-23 05:02:37 -04001572/*
Benny Halevye5e94012010-10-20 00:18:01 -04001573 * Layout segment is retreived from the server if not cached.
1574 * The appropriate layout segment is referenced and returned to the caller.
1575 */
Andy Adamson7c24d942011-06-13 18:22:38 -04001576struct pnfs_layout_segment *
Benny Halevye5e94012010-10-20 00:18:01 -04001577pnfs_update_layout(struct inode *ino,
1578 struct nfs_open_context *ctx,
Benny Halevyfb3296e2011-05-22 19:47:26 +03001579 loff_t pos,
1580 u64 count,
Trond Myklebusta75b9df2011-05-11 18:00:51 -04001581 enum pnfs_iomode iomode,
Tom Haynesc7d73af2016-05-25 07:31:14 -07001582 bool strict_iomode,
Trond Myklebusta75b9df2011-05-11 18:00:51 -04001583 gfp_t gfp_flags)
Benny Halevye5e94012010-10-20 00:18:01 -04001584{
Benny Halevyfb3296e2011-05-22 19:47:26 +03001585 struct pnfs_layout_range arg = {
1586 .iomode = iomode,
1587 .offset = pos,
1588 .length = count,
1589 };
Jeff Layton183d9e72016-05-17 12:28:47 -04001590 unsigned pg_offset, seq;
Weston Andros Adamson6382a442011-06-01 16:44:44 -04001591 struct nfs_server *server = NFS_SERVER(ino);
1592 struct nfs_client *clp = server->nfs_client;
Jeff Layton183d9e72016-05-17 12:28:47 -04001593 struct pnfs_layout_hdr *lo = NULL;
Benny Halevye5e94012010-10-20 00:18:01 -04001594 struct pnfs_layout_segment *lseg = NULL;
Jeff Layton183d9e72016-05-17 12:28:47 -04001595 nfs4_stateid stateid;
1596 long timeout = 0;
Trond Myklebust66b53f32016-07-14 14:28:31 -04001597 unsigned long giveup = jiffies + (clp->cl_lease_time << 1);
Weston Andros Adamson30005122013-02-28 20:30:10 -05001598 bool first;
Benny Halevye5e94012010-10-20 00:18:01 -04001599
Jeff Layton9a4bf312015-12-10 10:41:58 -05001600 if (!pnfs_enabled_sb(NFS_SERVER(ino))) {
Jeff Layton183d9e72016-05-17 12:28:47 -04001601 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
Jeff Layton9a4bf312015-12-10 10:41:58 -05001602 PNFS_UPDATE_LAYOUT_NO_PNFS);
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001603 goto out;
Jeff Layton9a4bf312015-12-10 10:41:58 -05001604 }
Andy Adamsond23d61c2012-05-23 05:02:37 -04001605
Jeff Layton9a4bf312015-12-10 10:41:58 -05001606 if (iomode == IOMODE_READ && i_size_read(ino) == 0) {
Jeff Layton183d9e72016-05-17 12:28:47 -04001607 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
Jeff Layton9a4bf312015-12-10 10:41:58 -05001608 PNFS_UPDATE_LAYOUT_RD_ZEROLEN);
Trond Myklebust4ae935602015-08-31 01:25:11 -07001609 goto out;
Jeff Layton9a4bf312015-12-10 10:41:58 -05001610 }
Trond Myklebust4ae935602015-08-31 01:25:11 -07001611
Jeff Layton9a4bf312015-12-10 10:41:58 -05001612 if (pnfs_within_mdsthreshold(ctx, ino, iomode)) {
Jeff Layton183d9e72016-05-17 12:28:47 -04001613 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
Jeff Layton9a4bf312015-12-10 10:41:58 -05001614 PNFS_UPDATE_LAYOUT_MDSTHRESH);
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001615 goto out;
Jeff Layton9a4bf312015-12-10 10:41:58 -05001616 }
Andy Adamsond23d61c2012-05-23 05:02:37 -04001617
Peng Tao9bf87482014-08-22 17:37:41 +08001618lookup_again:
Trond Myklebustb88fa692016-08-23 11:19:33 -04001619 nfs4_client_recover_expired_lease(clp);
Peng Tao9bf87482014-08-22 17:37:41 +08001620 first = false;
Benny Halevye5e94012010-10-20 00:18:01 -04001621 spin_lock(&ino->i_lock);
Peng Tao9fa40752011-07-30 20:52:32 -04001622 lo = pnfs_find_alloc_layout(ino, ctx, gfp_flags);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001623 if (lo == NULL) {
1624 spin_unlock(&ino->i_lock);
Jeff Layton183d9e72016-05-17 12:28:47 -04001625 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
Jeff Layton9a4bf312015-12-10 10:41:58 -05001626 PNFS_UPDATE_LAYOUT_NOMEM);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001627 goto out;
1628 }
Benny Halevye5e94012010-10-20 00:18:01 -04001629
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001630 /* Do we even need to bother with this? */
Trond Myklebusta59c30a2012-03-01 11:17:47 -05001631 if (test_bit(NFS_LAYOUT_BULK_RECALL, &lo->plh_flags)) {
Jeff Layton183d9e72016-05-17 12:28:47 -04001632 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
Jeff Layton9a4bf312015-12-10 10:41:58 -05001633 PNFS_UPDATE_LAYOUT_BULK_RECALL);
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001634 dprintk("%s matches recall, use MDS\n", __func__);
Benny Halevye5e94012010-10-20 00:18:01 -04001635 goto out_unlock;
1636 }
1637
1638 /* if LAYOUTGET already failed once we don't try again */
Trond Myklebust2e5b29f2015-12-14 16:25:11 -05001639 if (pnfs_layout_io_test_failed(lo, iomode)) {
Jeff Layton183d9e72016-05-17 12:28:47 -04001640 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
Jeff Layton9a4bf312015-12-10 10:41:58 -05001641 PNFS_UPDATE_LAYOUT_IO_TEST_FAIL);
Benny Halevye5e94012010-10-20 00:18:01 -04001642 goto out_unlock;
Jeff Layton9a4bf312015-12-10 10:41:58 -05001643 }
Benny Halevye5e94012010-10-20 00:18:01 -04001644
Tom Haynesc7d73af2016-05-25 07:31:14 -07001645 lseg = pnfs_find_lseg(lo, &arg, strict_iomode);
Jeff Layton183d9e72016-05-17 12:28:47 -04001646 if (lseg) {
1647 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1648 PNFS_UPDATE_LAYOUT_FOUND_CACHED);
1649 goto out_unlock;
1650 }
1651
1652 if (!nfs4_valid_open_stateid(ctx->state)) {
1653 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1654 PNFS_UPDATE_LAYOUT_INVALID_OPEN);
1655 goto out_unlock;
1656 }
1657
1658 /*
1659 * Choose a stateid for the LAYOUTGET. If we don't have a layout
1660 * stateid, or it has been invalidated, then we must use the open
1661 * stateid.
1662 */
Trond Myklebust67a3b722016-06-17 16:48:19 -04001663 if (test_bit(NFS_LAYOUT_INVALID_STID, &lo->plh_flags)) {
Jeff Layton183d9e72016-05-17 12:28:47 -04001664
1665 /*
1666 * The first layoutget for the file. Need to serialize per
Peng Tao9bf87482014-08-22 17:37:41 +08001667 * RFC 5661 Errata 3208.
1668 */
1669 if (test_and_set_bit(NFS_LAYOUT_FIRST_LAYOUTGET,
1670 &lo->plh_flags)) {
1671 spin_unlock(&ino->i_lock);
1672 wait_on_bit(&lo->plh_flags, NFS_LAYOUT_FIRST_LAYOUTGET,
1673 TASK_UNINTERRUPTIBLE);
1674 pnfs_put_layout_hdr(lo);
Jeff Layton183d9e72016-05-17 12:28:47 -04001675 dprintk("%s retrying\n", __func__);
Peng Tao9bf87482014-08-22 17:37:41 +08001676 goto lookup_again;
1677 }
Jeff Layton183d9e72016-05-17 12:28:47 -04001678
1679 first = true;
1680 do {
1681 seq = read_seqbegin(&ctx->state->seqlock);
1682 nfs4_stateid_copy(&stateid, &ctx->state->stateid);
1683 } while (read_seqretry(&ctx->state->seqlock, seq));
Peng Tao9bf87482014-08-22 17:37:41 +08001684 } else {
Jeff Layton183d9e72016-05-17 12:28:47 -04001685 nfs4_stateid_copy(&stateid, &lo->plh_stateid);
Peng Tao9bf87482014-08-22 17:37:41 +08001686 }
Andy Adamson568e8c42011-03-01 01:34:22 +00001687
Peng Taoaa8a45e2014-12-01 08:22:23 +08001688 /*
1689 * Because we free lsegs before sending LAYOUTRETURN, we need to wait
1690 * for LAYOUTRETURN even if first is true.
1691 */
Trond Myklebust8f70f532015-08-04 16:09:44 -04001692 if (test_bit(NFS_LAYOUT_RETURN, &lo->plh_flags)) {
Peng Taoaa8a45e2014-12-01 08:22:23 +08001693 spin_unlock(&ino->i_lock);
1694 dprintk("%s wait for layoutreturn\n", __func__);
1695 if (pnfs_prepare_to_retry_layoutget(lo)) {
Tom Haynesd67ae822014-12-11 17:02:04 -05001696 if (first)
1697 pnfs_clear_first_layoutget(lo);
Peng Taoaa8a45e2014-12-01 08:22:23 +08001698 pnfs_put_layout_hdr(lo);
1699 dprintk("%s retrying\n", __func__);
Jeff Layton183d9e72016-05-17 12:28:47 -04001700 trace_pnfs_update_layout(ino, pos, count, iomode, lo,
1701 lseg, PNFS_UPDATE_LAYOUT_RETRY);
Peng Taoaa8a45e2014-12-01 08:22:23 +08001702 goto lookup_again;
1703 }
Jeff Layton183d9e72016-05-17 12:28:47 -04001704 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
Jeff Layton9a4bf312015-12-10 10:41:58 -05001705 PNFS_UPDATE_LAYOUT_RETURN);
Peng Taoaa8a45e2014-12-01 08:22:23 +08001706 goto out_put_layout_hdr;
1707 }
1708
Jeff Layton9a4bf312015-12-10 10:41:58 -05001709 if (pnfs_layoutgets_blocked(lo)) {
Jeff Layton183d9e72016-05-17 12:28:47 -04001710 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
Jeff Layton9a4bf312015-12-10 10:41:58 -05001711 PNFS_UPDATE_LAYOUT_BLOCKED);
Fred Isamancf7d63f2011-01-06 11:36:25 +00001712 goto out_unlock;
Jeff Layton9a4bf312015-12-10 10:41:58 -05001713 }
Fred Isamancf7d63f2011-01-06 11:36:25 +00001714 atomic_inc(&lo->plh_outstanding);
Fred Isamanf49f9ba2011-02-03 18:28:52 +00001715 spin_unlock(&ino->i_lock);
Weston Andros Adamson30005122013-02-28 20:30:10 -05001716
Peng Taoabb9a002014-08-22 17:37:40 +08001717 if (list_empty(&lo->plh_layouts)) {
Fred Isaman2130ff62011-01-06 11:36:26 +00001718 /* The lo must be on the clp list if there is any
1719 * chance of a CB_LAYOUTRECALL(FILE) coming in.
1720 */
1721 spin_lock(&clp->cl_lock);
Peng Taoabb9a002014-08-22 17:37:40 +08001722 if (list_empty(&lo->plh_layouts))
1723 list_add_tail(&lo->plh_layouts, &server->layouts);
Fred Isaman2130ff62011-01-06 11:36:26 +00001724 spin_unlock(&clp->cl_lock);
1725 }
Benny Halevye5e94012010-10-20 00:18:01 -04001726
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001727 pg_offset = arg.offset & ~PAGE_MASK;
Benny Halevy707ed5f2011-05-22 19:47:46 +03001728 if (pg_offset) {
1729 arg.offset -= pg_offset;
1730 arg.length += pg_offset;
1731 }
Andy Adamson7c24d942011-06-13 18:22:38 -04001732 if (arg.length != NFS4_MAX_UINT64)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001733 arg.length = PAGE_ALIGN(arg.length);
Benny Halevy707ed5f2011-05-22 19:47:46 +03001734
Jeff Layton183d9e72016-05-17 12:28:47 -04001735 lseg = send_layoutget(lo, ctx, &stateid, &arg, &timeout, gfp_flags);
1736 trace_pnfs_update_layout(ino, pos, count, iomode, lo, lseg,
1737 PNFS_UPDATE_LAYOUT_SEND_LAYOUTGET);
Trond Myklebust56b38a12016-07-14 18:34:12 -04001738 atomic_dec(&lo->plh_outstanding);
Jeff Layton83026d82016-05-17 12:28:46 -04001739 if (IS_ERR(lseg)) {
Jeff Layton183d9e72016-05-17 12:28:47 -04001740 switch(PTR_ERR(lseg)) {
Trond Myklebuste85d7ee2016-07-14 18:46:24 -04001741 case -EBUSY:
Jeff Layton183d9e72016-05-17 12:28:47 -04001742 if (time_after(jiffies, giveup))
1743 lseg = NULL;
Trond Myklebust66b53f32016-07-14 14:28:31 -04001744 break;
1745 case -ERECALLCONFLICT:
1746 /* Huh? We hold no layouts, how is there a recall? */
1747 if (first) {
1748 lseg = NULL;
1749 break;
1750 }
1751 /* Destroy the existing layout and start over */
1752 if (time_after(jiffies, giveup))
1753 pnfs_destroy_layout(NFS_I(ino));
Jeff Layton183d9e72016-05-17 12:28:47 -04001754 /* Fallthrough */
1755 case -EAGAIN:
Trond Myklebust56b38a12016-07-14 18:34:12 -04001756 break;
Jeff Layton183d9e72016-05-17 12:28:47 -04001757 default:
1758 if (!nfs_error_is_fatal(PTR_ERR(lseg))) {
1759 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
1760 lseg = NULL;
1761 }
Trond Myklebust56b38a12016-07-14 18:34:12 -04001762 goto out_put_layout_hdr;
1763 }
1764 if (lseg) {
1765 if (first)
1766 pnfs_clear_first_layoutget(lo);
1767 trace_pnfs_update_layout(ino, pos, count,
1768 iomode, lo, lseg, PNFS_UPDATE_LAYOUT_RETRY);
1769 pnfs_put_layout_hdr(lo);
1770 goto lookup_again;
Jeff Layton83026d82016-05-17 12:28:46 -04001771 }
1772 } else {
1773 pnfs_layout_clear_fail_bit(lo, pnfs_iomode_to_fail_bit(iomode));
1774 }
1775
Trond Myklebust830ffb52012-09-20 21:25:19 -04001776out_put_layout_hdr:
Tom Haynesd67ae822014-12-11 17:02:04 -05001777 if (first)
1778 pnfs_clear_first_layoutget(lo);
Trond Myklebust70c3bd22012-09-18 20:51:13 -04001779 pnfs_put_layout_hdr(lo);
Benny Halevye5e94012010-10-20 00:18:01 -04001780out:
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001781 dprintk("%s: inode %s/%llu pNFS layout segment %s for "
1782 "(%s, offset: %llu, length: %llu)\n",
1783 __func__, ino->i_sb->s_id,
1784 (unsigned long long)NFS_FILEID(ino),
Peng Taod600ad12015-12-04 02:57:48 +08001785 IS_ERR_OR_NULL(lseg) ? "not found" : "found",
Trond Myklebustf86bbcf2012-09-26 11:21:40 -04001786 iomode==IOMODE_RW ? "read/write" : "read-only",
1787 (unsigned long long)pos,
1788 (unsigned long long)count);
Benny Halevye5e94012010-10-20 00:18:01 -04001789 return lseg;
1790out_unlock:
1791 spin_unlock(&ino->i_lock);
Trond Myklebust830ffb52012-09-20 21:25:19 -04001792 goto out_put_layout_hdr;
Benny Halevye5e94012010-10-20 00:18:01 -04001793}
Andy Adamson7c24d942011-06-13 18:22:38 -04001794EXPORT_SYMBOL_GPL(pnfs_update_layout);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001795
Trond Myklebust540d9862015-08-25 11:16:13 -04001796static bool
1797pnfs_sanity_check_layout_range(struct pnfs_layout_range *range)
1798{
1799 switch (range->iomode) {
1800 case IOMODE_READ:
1801 case IOMODE_RW:
1802 break;
1803 default:
1804 return false;
1805 }
1806 if (range->offset == NFS4_MAX_UINT64)
1807 return false;
1808 if (range->length == 0)
1809 return false;
1810 if (range->length != NFS4_MAX_UINT64 &&
1811 range->length > NFS4_MAX_UINT64 - range->offset)
1812 return false;
1813 return true;
1814}
1815
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -04001816struct pnfs_layout_segment *
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001817pnfs_layout_process(struct nfs4_layoutget *lgp)
1818{
1819 struct pnfs_layout_hdr *lo = NFS_I(lgp->args.inode)->layout;
1820 struct nfs4_layoutget_res *res = &lgp->res;
1821 struct pnfs_layout_segment *lseg;
Fred Isamanb7edfaa2011-01-06 11:36:21 +00001822 struct inode *ino = lo->plh_inode;
Trond Myklebust78096cc2014-02-12 10:02:27 -05001823 LIST_HEAD(free_me);
Trond Myklebust540d9862015-08-25 11:16:13 -04001824
1825 if (!pnfs_sanity_check_layout_range(&res->range))
Jeff Layton1b3c6d02016-05-17 12:28:48 -04001826 return ERR_PTR(-EINVAL);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001827
1828 /* Inject layout blob into I/O device driver */
Trond Myklebusta75b9df2011-05-11 18:00:51 -04001829 lseg = NFS_SERVER(ino)->pnfs_curr_ld->alloc_lseg(lo, res, lgp->gfp_flags);
Jeff Layton1b3c6d02016-05-17 12:28:48 -04001830 if (IS_ERR_OR_NULL(lseg)) {
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001831 if (!lseg)
Jeff Layton1b3c6d02016-05-17 12:28:48 -04001832 lseg = ERR_PTR(-ENOMEM);
1833
1834 dprintk("%s: Could not allocate layout: error %ld\n",
1835 __func__, PTR_ERR(lseg));
1836 return lseg;
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001837 }
1838
Trond Myklebust119cef972016-07-24 15:10:12 -04001839 pnfs_init_lseg(lo, lseg, &res->range, &res->stateid);
Christoph Hellwig1013df612014-08-21 11:09:18 -05001840
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001841 spin_lock(&ino->i_lock);
Trond Myklebuste1c06f82015-08-04 16:40:08 -04001842 if (pnfs_layoutgets_blocked(lo)) {
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001843 dprintk("%s forget reply due to state\n", __func__);
Jeff Layton1b3c6d02016-05-17 12:28:48 -04001844 goto out_forget;
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001845 }
Trond Myklebust038d6492012-10-02 16:38:41 -07001846
Christoph Hellwig362f7472014-08-21 11:09:20 -05001847 if (nfs4_stateid_match_other(&lo->plh_stateid, &res->stateid)) {
1848 /* existing state ID, make sure the sequence number matches. */
1849 if (pnfs_layout_stateid_blocked(lo, &res->stateid)) {
1850 dprintk("%s forget reply due to sequence\n", __func__);
Jeff Layton1b3c6d02016-05-17 12:28:48 -04001851 goto out_forget;
Christoph Hellwig362f7472014-08-21 11:09:20 -05001852 }
1853 pnfs_set_layout_stateid(lo, &res->stateid, false);
1854 } else {
1855 /*
1856 * We got an entirely new state ID. Mark all segments for the
1857 * inode invalid, and don't bother validating the stateid
1858 * sequence number.
1859 */
Trond Myklebustd9b61702016-07-24 15:04:07 -04001860 pnfs_mark_layout_stateid_invalid(lo, &free_me);
Christoph Hellwig362f7472014-08-21 11:09:20 -05001861
Trond Myklebust2a59a042016-09-03 11:20:04 -04001862 pnfs_set_layout_stateid(lo, &res->stateid, true);
Christoph Hellwig362f7472014-08-21 11:09:20 -05001863 }
Trond Myklebust038d6492012-10-02 16:38:41 -07001864
Trond Myklebust9369a432012-09-18 20:57:08 -04001865 pnfs_get_lseg(lseg);
Trond Myklebust03772d22015-08-25 08:54:17 -04001866 pnfs_layout_insert_lseg(lo, lseg, &free_me);
Trond Myklebust8e0acf92016-07-21 11:53:29 -04001867
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001868
Peng Tao39761432015-08-21 12:49:44 +08001869 if (res->return_on_close)
Fred Isamanf7e89172011-01-06 11:36:32 +00001870 set_bit(NFS_LSEG_ROC, &lseg->pls_flags);
Fred Isamanf7e89172011-01-06 11:36:32 +00001871
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001872 spin_unlock(&ino->i_lock);
Trond Myklebust78096cc2014-02-12 10:02:27 -05001873 pnfs_free_lseg_list(&free_me);
Trond Myklebusta0b0a6e2012-09-17 17:12:15 -04001874 return lseg;
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001875
Jeff Layton1b3c6d02016-05-17 12:28:48 -04001876out_forget:
Fred Isaman43f1b3d2011-01-06 11:36:30 +00001877 spin_unlock(&ino->i_lock);
1878 lseg->pls_layout = lo;
1879 NFS_SERVER(ino)->pnfs_curr_ld->free_lseg(lseg);
Jeff Layton1b3c6d02016-05-17 12:28:48 -04001880 return ERR_PTR(-EAGAIN);
Andy Adamsonb1f69b72010-10-20 00:18:03 -04001881}
1882
Peng Tao016256d2014-09-06 00:53:23 +08001883static void
Jeff Layton3982a6a2016-05-17 12:28:41 -04001884pnfs_set_plh_return_info(struct pnfs_layout_hdr *lo, enum pnfs_iomode iomode,
1885 u32 seq)
Trond Myklebust5c97f5d2016-01-04 11:53:04 -05001886{
Trond Myklebust2d6cf5a2016-07-21 13:06:18 -04001887 if (lo->plh_return_iomode != 0 && lo->plh_return_iomode != iomode)
Trond Myklebust5c97f5d2016-01-04 11:53:04 -05001888 iomode = IOMODE_ANY;
1889 lo->plh_return_iomode = iomode;
Trond Myklebuste0fa0d02016-02-15 12:56:17 -05001890 set_bit(NFS_LAYOUT_RETURN_REQUESTED, &lo->plh_flags);
Trond Myklebust2d6cf5a2016-07-21 13:06:18 -04001891 if (seq != 0) {
1892 WARN_ON_ONCE(lo->plh_return_seq != 0 && lo->plh_return_seq != seq);
Jeff Layton3982a6a2016-05-17 12:28:41 -04001893 lo->plh_return_seq = seq;
Trond Myklebust2d6cf5a2016-07-21 13:06:18 -04001894 }
Trond Myklebust5c97f5d2016-01-04 11:53:04 -05001895}
1896
Trond Myklebust2f215962016-02-15 12:36:04 -05001897/**
1898 * pnfs_mark_matching_lsegs_return - Free or return matching layout segments
1899 * @lo: pointer to layout header
1900 * @tmp_list: list header to be used with pnfs_free_lseg_list()
1901 * @return_range: describe layout segment ranges to be returned
1902 *
1903 * This function is mainly intended for use by layoutrecall. It attempts
1904 * to free the layout segment immediately, or else to mark it for return
1905 * as soon as its reference count drops to zero.
1906 */
Trond Myklebust10335552016-01-04 11:23:52 -05001907int
Peng Tao016256d2014-09-06 00:53:23 +08001908pnfs_mark_matching_lsegs_return(struct pnfs_layout_hdr *lo,
1909 struct list_head *tmp_list,
Jeff Layton6d597e12016-05-17 12:28:42 -04001910 const struct pnfs_layout_range *return_range,
1911 u32 seq)
Peng Tao016256d2014-09-06 00:53:23 +08001912{
1913 struct pnfs_layout_segment *lseg, *next;
Trond Myklebust10335552016-01-04 11:23:52 -05001914 int remaining = 0;
Peng Tao016256d2014-09-06 00:53:23 +08001915
1916 dprintk("%s:Begin lo %p\n", __func__, lo);
1917
1918 if (list_empty(&lo->plh_segs))
Trond Myklebust10335552016-01-04 11:23:52 -05001919 return 0;
Peng Tao016256d2014-09-06 00:53:23 +08001920
Trond Myklebustfc7ff362015-12-28 10:28:59 -05001921 assert_spin_locked(&lo->plh_inode->i_lock);
Peng Tao016256d2014-09-06 00:53:23 +08001922
1923 list_for_each_entry_safe(lseg, next, &lo->plh_segs, pls_list)
Trond Myklebuste036f462016-07-22 11:13:22 -04001924 if (pnfs_match_lseg_recall(lseg, return_range, seq)) {
Peng Tao016256d2014-09-06 00:53:23 +08001925 dprintk("%s: marking lseg %p iomode %d "
1926 "offset %llu length %llu\n", __func__,
1927 lseg, lseg->pls_range.iomode,
1928 lseg->pls_range.offset,
1929 lseg->pls_range.length);
Trond Myklebust2f215962016-02-15 12:36:04 -05001930 if (mark_lseg_invalid(lseg, tmp_list))
1931 continue;
1932 remaining++;
Peng Tao016256d2014-09-06 00:53:23 +08001933 set_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags);
Peng Tao016256d2014-09-06 00:53:23 +08001934 }
Jeff Layton6d597e12016-05-17 12:28:42 -04001935
1936 if (remaining)
1937 pnfs_set_plh_return_info(lo, return_range->iomode, seq);
1938
Trond Myklebust10335552016-01-04 11:23:52 -05001939 return remaining;
Peng Tao016256d2014-09-06 00:53:23 +08001940}
1941
1942void pnfs_error_mark_layout_for_return(struct inode *inode,
1943 struct pnfs_layout_segment *lseg)
1944{
1945 struct pnfs_layout_hdr *lo = NFS_I(inode)->layout;
Peng Tao016256d2014-09-06 00:53:23 +08001946 struct pnfs_layout_range range = {
1947 .iomode = lseg->pls_range.iomode,
1948 .offset = 0,
1949 .length = NFS4_MAX_UINT64,
1950 };
1951 LIST_HEAD(free_me);
Trond Myklebust10335552016-01-04 11:23:52 -05001952 bool return_now = false;
Peng Tao016256d2014-09-06 00:53:23 +08001953
1954 spin_lock(&inode->i_lock);
Trond Myklebust2d6cf5a2016-07-21 13:06:18 -04001955 pnfs_set_plh_return_info(lo, range.iomode, 0);
Trond Myklebust24b049f2016-10-17 17:54:32 -04001956 /* Block LAYOUTGET */
1957 set_bit(NFS_LAYOUT_RETURN, &lo->plh_flags);
Peng Tao016256d2014-09-06 00:53:23 +08001958 /*
1959 * mark all matching lsegs so that we are sure to have no live
1960 * segments at hand when sending layoutreturn. See pnfs_put_lseg()
1961 * for how it works.
1962 */
Trond Myklebust2d6cf5a2016-07-21 13:06:18 -04001963 if (!pnfs_mark_matching_lsegs_return(lo, &free_me, &range, 0)) {
Trond Myklebust10335552016-01-04 11:23:52 -05001964 nfs4_stateid stateid;
Trond Myklebuste5fd1902016-07-21 12:44:15 -04001965 enum pnfs_iomode iomode;
Trond Myklebust10335552016-01-04 11:23:52 -05001966
Trond Myklebuste5fd1902016-07-21 12:44:15 -04001967 return_now = pnfs_prepare_layoutreturn(lo, &stateid, &iomode);
Trond Myklebust10335552016-01-04 11:23:52 -05001968 spin_unlock(&inode->i_lock);
1969 if (return_now)
1970 pnfs_send_layoutreturn(lo, &stateid, iomode, false);
1971 } else {
1972 spin_unlock(&inode->i_lock);
1973 nfs_commit_inode(inode, 0);
1974 }
Peng Tao016256d2014-09-06 00:53:23 +08001975 pnfs_free_lseg_list(&free_me);
1976}
1977EXPORT_SYMBOL_GPL(pnfs_error_mark_layout_for_return);
1978
Trond Myklebustd8007d42011-06-10 13:30:23 -04001979void
1980pnfs_generic_pg_init_read(struct nfs_pageio_descriptor *pgio, struct nfs_page *req)
1981{
Peng Tao1fd937b2012-09-25 14:55:57 +08001982 u64 rd_size = req->wb_bytes;
1983
Peng Taocb5d04b2015-01-24 22:14:52 +08001984 if (pgio->pg_lseg == NULL) {
1985 if (pgio->pg_dreq == NULL)
1986 rd_size = i_size_read(pgio->pg_inode) - req_offset(req);
1987 else
1988 rd_size = nfs_dreq_bytes_left(pgio->pg_dreq);
Trond Myklebustd8007d42011-06-10 13:30:23 -04001989
Peng Taocb5d04b2015-01-24 22:14:52 +08001990 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
1991 req->wb_context,
1992 req_offset(req),
1993 rd_size,
1994 IOMODE_READ,
Tom Haynesc7d73af2016-05-25 07:31:14 -07001995 false,
Peng Taocb5d04b2015-01-24 22:14:52 +08001996 GFP_KERNEL);
Peng Taod600ad12015-12-04 02:57:48 +08001997 if (IS_ERR(pgio->pg_lseg)) {
1998 pgio->pg_error = PTR_ERR(pgio->pg_lseg);
1999 pgio->pg_lseg = NULL;
2000 return;
2001 }
Peng Taocb5d04b2015-01-24 22:14:52 +08002002 }
Trond Myklebuste885de12011-06-10 13:30:23 -04002003 /* If no lseg, fall back to read through mds */
2004 if (pgio->pg_lseg == NULL)
Trond Myklebust1f945352011-07-13 15:59:57 -04002005 nfs_pageio_reset_read_mds(pgio);
Trond Myklebuste885de12011-06-10 13:30:23 -04002006
Trond Myklebustd8007d42011-06-10 13:30:23 -04002007}
2008EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_read);
2009
2010void
Peng Tao62965562012-09-25 14:55:57 +08002011pnfs_generic_pg_init_write(struct nfs_pageio_descriptor *pgio,
2012 struct nfs_page *req, u64 wb_size)
Trond Myklebustd8007d42011-06-10 13:30:23 -04002013{
Peng Taod600ad12015-12-04 02:57:48 +08002014 if (pgio->pg_lseg == NULL) {
Peng Taocb5d04b2015-01-24 22:14:52 +08002015 pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
2016 req->wb_context,
2017 req_offset(req),
2018 wb_size,
2019 IOMODE_RW,
Tom Haynesc7d73af2016-05-25 07:31:14 -07002020 false,
Peng Taocb5d04b2015-01-24 22:14:52 +08002021 GFP_NOFS);
Peng Taod600ad12015-12-04 02:57:48 +08002022 if (IS_ERR(pgio->pg_lseg)) {
2023 pgio->pg_error = PTR_ERR(pgio->pg_lseg);
2024 pgio->pg_lseg = NULL;
2025 return;
2026 }
2027 }
Trond Myklebuste885de12011-06-10 13:30:23 -04002028 /* If no lseg, fall back to write through mds */
2029 if (pgio->pg_lseg == NULL)
Trond Myklebust1f945352011-07-13 15:59:57 -04002030 nfs_pageio_reset_write_mds(pgio);
Trond Myklebustd8007d42011-06-10 13:30:23 -04002031}
2032EXPORT_SYMBOL_GPL(pnfs_generic_pg_init_write);
2033
Weston Andros Adamson180bb5e2014-09-10 15:48:01 -04002034void
2035pnfs_generic_pg_cleanup(struct nfs_pageio_descriptor *desc)
2036{
2037 if (desc->pg_lseg) {
2038 pnfs_put_lseg(desc->pg_lseg);
2039 desc->pg_lseg = NULL;
2040 }
2041}
2042EXPORT_SYMBOL_GPL(pnfs_generic_pg_cleanup);
2043
Weston Andros Adamsonb4fdac12014-05-15 11:56:43 -04002044/*
2045 * Return 0 if @req cannot be coalesced into @pgio, otherwise return the number
2046 * of bytes (maximum @req->wb_bytes) that can be coalesced.
2047 */
2048size_t
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04002049pnfs_generic_pg_test(struct nfs_pageio_descriptor *pgio,
2050 struct nfs_page *prev, struct nfs_page *req)
Fred Isamanbae724e2011-03-01 01:34:15 +00002051{
Weston Andros Adamson0f9c4292014-05-15 11:56:51 -04002052 unsigned int size;
Weston Andros Adamsonc5e20cb2014-06-09 17:47:26 -04002053 u64 seg_end, req_start, seg_left;
Weston Andros Adamson0f9c4292014-05-15 11:56:51 -04002054
2055 size = nfs_generic_pg_test(pgio, prev, req);
Weston Andros Adamson0f9c4292014-05-15 11:56:51 -04002056 if (!size)
2057 return 0;
Benny Halevy89a58e32011-05-25 20:54:40 +03002058
Trond Myklebust19982ba2011-06-10 13:30:23 -04002059 /*
Weston Andros Adamsonc5e20cb2014-06-09 17:47:26 -04002060 * 'size' contains the number of bytes left in the current page (up
2061 * to the original size asked for in @req->wb_bytes).
2062 *
2063 * Calculate how many bytes are left in the layout segment
2064 * and if there are less bytes than 'size', return that instead.
Trond Myklebust19982ba2011-06-10 13:30:23 -04002065 *
2066 * Please also note that 'end_offset' is actually the offset of the
2067 * first byte that lies outside the pnfs_layout_range. FIXME?
2068 *
2069 */
Weston Andros Adamson19b54842014-05-15 11:56:55 -04002070 if (pgio->pg_lseg) {
Weston Andros Adamsonc5e20cb2014-06-09 17:47:26 -04002071 seg_end = end_offset(pgio->pg_lseg->pls_range.offset,
2072 pgio->pg_lseg->pls_range.length);
2073 req_start = req_offset(req);
Weston Andros Adamson7c137892015-01-30 11:01:02 -05002074 WARN_ON_ONCE(req_start >= seg_end);
Weston Andros Adamsonc5e20cb2014-06-09 17:47:26 -04002075 /* start of request is past the last byte of this segment */
Weston Andros Adamson7c137892015-01-30 11:01:02 -05002076 if (req_start >= seg_end) {
2077 /* reference the new lseg */
2078 if (pgio->pg_ops->pg_cleanup)
2079 pgio->pg_ops->pg_cleanup(pgio);
2080 if (pgio->pg_ops->pg_init)
2081 pgio->pg_ops->pg_init(pgio, req);
Weston Andros Adamson19b54842014-05-15 11:56:55 -04002082 return 0;
Weston Andros Adamson7c137892015-01-30 11:01:02 -05002083 }
Weston Andros Adamsonc5e20cb2014-06-09 17:47:26 -04002084
2085 /* adjust 'size' iff there are fewer bytes left in the
2086 * segment than what nfs_generic_pg_test returned */
2087 seg_left = seg_end - req_start;
2088 if (seg_left < size)
2089 size = (unsigned int)seg_left;
Weston Andros Adamson19b54842014-05-15 11:56:55 -04002090 }
Weston Andros Adamson0f9c4292014-05-15 11:56:51 -04002091
Weston Andros Adamson19b54842014-05-15 11:56:55 -04002092 return size;
Fred Isamanbae724e2011-03-01 01:34:15 +00002093}
Benny Halevy89a58e32011-05-25 20:54:40 +03002094EXPORT_SYMBOL_GPL(pnfs_generic_pg_test);
Fred Isamanbae724e2011-03-01 01:34:15 +00002095
Weston Andros Adamson53113ad2014-06-09 11:48:38 -04002096int pnfs_write_done_resend_to_mds(struct nfs_pgio_header *hdr)
Trond Myklebuste2fecb22012-01-06 08:57:46 -05002097{
2098 struct nfs_pageio_descriptor pgio;
Trond Myklebuste2fecb22012-01-06 08:57:46 -05002099
2100 /* Resend all requests through the MDS */
Weston Andros Adamson53113ad2014-06-09 11:48:38 -04002101 nfs_pageio_init_write(&pgio, hdr->inode, FLUSH_STABLE, true,
2102 hdr->completion_ops);
Trond Myklebustc7070112015-06-17 19:56:22 -04002103 set_bit(NFS_CONTEXT_RESEND_WRITES, &hdr->args.context->flags);
Weston Andros Adamson53113ad2014-06-09 11:48:38 -04002104 return nfs_pageio_resend(&pgio, hdr);
Trond Myklebuste2fecb22012-01-06 08:57:46 -05002105}
Andy Adamsone7dd79af2012-04-27 17:53:46 -04002106EXPORT_SYMBOL_GPL(pnfs_write_done_resend_to_mds);
Trond Myklebuste2fecb22012-01-06 08:57:46 -05002107
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002108static void pnfs_ld_handle_write_error(struct nfs_pgio_header *hdr)
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002109{
Fred Isamancd841602012-04-20 14:47:44 -04002110
2111 dprintk("pnfs write error = %d\n", hdr->pnfs_error);
2112 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002113 PNFS_LAYOUTRET_ON_ERROR) {
Fred Isamancd841602012-04-20 14:47:44 -04002114 pnfs_return_layout(hdr->inode);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002115 }
Fred Isaman6c75dc02012-04-20 14:47:47 -04002116 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
Weston Andros Adamson53113ad2014-06-09 11:48:38 -04002117 hdr->task.tk_status = pnfs_write_done_resend_to_mds(hdr);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002118}
2119
Benny Halevyd20581a2011-05-22 19:52:03 +03002120/*
2121 * Called by non rpc-based layout drivers
2122 */
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002123void pnfs_ld_write_done(struct nfs_pgio_header *hdr)
Fred Isaman94ad1c82011-03-01 01:34:14 +00002124{
Kinglong Meef8417b482015-10-16 17:23:29 +08002125 if (likely(!hdr->pnfs_error)) {
Trond Myklebust67af7612015-03-25 20:40:38 -04002126 pnfs_set_layoutcommit(hdr->inode, hdr->lseg,
2127 hdr->mds_offset + hdr->res.count);
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002128 hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
Kinglong Meef8417b482015-10-16 17:23:29 +08002129 }
2130 trace_nfs4_pnfs_write(hdr, hdr->pnfs_error);
2131 if (unlikely(hdr->pnfs_error))
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002132 pnfs_ld_handle_write_error(hdr);
2133 hdr->mds_ops->rpc_release(hdr);
Fred Isaman44b83792011-03-03 15:13:44 +00002134}
Benny Halevyd20581a2011-05-22 19:52:03 +03002135EXPORT_SYMBOL_GPL(pnfs_ld_write_done);
Fred Isaman44b83792011-03-03 15:13:44 +00002136
Trond Myklebustdce81292011-07-13 15:59:19 -04002137static void
2138pnfs_write_through_mds(struct nfs_pageio_descriptor *desc,
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002139 struct nfs_pgio_header *hdr)
Trond Myklebustdce81292011-07-13 15:59:19 -04002140{
Peng Tao48d635f2014-11-10 08:35:35 +08002141 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04002142
Fred Isaman6c75dc02012-04-20 14:47:47 -04002143 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04002144 list_splice_tail_init(&hdr->pages, &mirror->pg_list);
Fred Isaman6c75dc02012-04-20 14:47:47 -04002145 nfs_pageio_reset_write_mds(desc);
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04002146 mirror->pg_recoalesce = 1;
Fred Isaman6c75dc02012-04-20 14:47:47 -04002147 }
Trond Myklebust1ca018d2015-06-17 19:41:51 -04002148 hdr->release(hdr);
Trond Myklebustdce81292011-07-13 15:59:19 -04002149}
2150
2151static enum pnfs_try_status
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002152pnfs_try_to_write_data(struct nfs_pgio_header *hdr,
Trond Myklebustdce81292011-07-13 15:59:19 -04002153 const struct rpc_call_ops *call_ops,
2154 struct pnfs_layout_segment *lseg,
2155 int how)
Andy Adamson0382b742011-03-03 15:13:45 +00002156{
Fred Isamancd841602012-04-20 14:47:44 -04002157 struct inode *inode = hdr->inode;
Andy Adamson0382b742011-03-03 15:13:45 +00002158 enum pnfs_try_status trypnfs;
2159 struct nfs_server *nfss = NFS_SERVER(inode);
2160
Fred Isamancd841602012-04-20 14:47:44 -04002161 hdr->mds_ops = call_ops;
Andy Adamson0382b742011-03-03 15:13:45 +00002162
2163 dprintk("%s: Writing ino:%lu %u@%llu (how %d)\n", __func__,
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002164 inode->i_ino, hdr->args.count, hdr->args.offset, how);
2165 trypnfs = nfss->pnfs_curr_ld->write_pagelist(hdr, how);
Fred Isaman6c75dc02012-04-20 14:47:47 -04002166 if (trypnfs != PNFS_NOT_ATTEMPTED)
Andy Adamson0382b742011-03-03 15:13:45 +00002167 nfs_inc_stats(inode, NFSIOS_PNFS_WRITE);
Andy Adamson0382b742011-03-03 15:13:45 +00002168 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
2169 return trypnfs;
2170}
2171
Trond Myklebustdce81292011-07-13 15:59:19 -04002172static void
Weston Andros Adamson7f714722014-05-15 11:56:53 -04002173pnfs_do_write(struct nfs_pageio_descriptor *desc,
2174 struct nfs_pgio_header *hdr, int how)
Trond Myklebustdce81292011-07-13 15:59:19 -04002175{
Trond Myklebustdce81292011-07-13 15:59:19 -04002176 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
2177 struct pnfs_layout_segment *lseg = desc->pg_lseg;
Weston Andros Adamson7f714722014-05-15 11:56:53 -04002178 enum pnfs_try_status trypnfs;
Trond Myklebustdce81292011-07-13 15:59:19 -04002179
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002180 trypnfs = pnfs_try_to_write_data(hdr, call_ops, lseg, how);
Weston Andros Adamson7f714722014-05-15 11:56:53 -04002181 if (trypnfs == PNFS_NOT_ATTEMPTED)
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002182 pnfs_write_through_mds(desc, hdr);
Trond Myklebustdce81292011-07-13 15:59:19 -04002183}
2184
Fred Isaman6c75dc02012-04-20 14:47:47 -04002185static void pnfs_writehdr_free(struct nfs_pgio_header *hdr)
2186{
Trond Myklebust9369a432012-09-18 20:57:08 -04002187 pnfs_put_lseg(hdr->lseg);
Weston Andros Adamson1e7f3a42014-06-09 11:48:33 -04002188 nfs_pgio_header_free(hdr);
Fred Isaman6c75dc02012-04-20 14:47:47 -04002189}
2190
Trond Myklebustdce81292011-07-13 15:59:19 -04002191int
2192pnfs_generic_pg_writepages(struct nfs_pageio_descriptor *desc)
2193{
Fred Isaman6c75dc02012-04-20 14:47:47 -04002194 struct nfs_pgio_header *hdr;
Trond Myklebustdce81292011-07-13 15:59:19 -04002195 int ret;
2196
Weston Andros Adamson1e7f3a42014-06-09 11:48:33 -04002197 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
2198 if (!hdr) {
Peng Tao2bff2282015-12-05 02:03:17 +08002199 desc->pg_error = -ENOMEM;
2200 return desc->pg_error;
Fred Isaman6c75dc02012-04-20 14:47:47 -04002201 }
Fred Isaman6c75dc02012-04-20 14:47:47 -04002202 nfs_pgheader_init(desc, hdr, pnfs_writehdr_free);
Weston Andros Adamson180bb5e2014-09-10 15:48:01 -04002203
Trond Myklebust9369a432012-09-18 20:57:08 -04002204 hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
Anna Schumakeref2c4882014-05-06 09:12:36 -04002205 ret = nfs_generic_pgio(desc, hdr);
Weston Andros Adamson180bb5e2014-09-10 15:48:01 -04002206 if (!ret)
Weston Andros Adamson7f714722014-05-15 11:56:53 -04002207 pnfs_do_write(desc, hdr, desc->pg_ioflags);
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04002208
Fred Isaman6c75dc02012-04-20 14:47:47 -04002209 return ret;
Trond Myklebustdce81292011-07-13 15:59:19 -04002210}
2211EXPORT_SYMBOL_GPL(pnfs_generic_pg_writepages);
2212
Weston Andros Adamson53113ad2014-06-09 11:48:38 -04002213int pnfs_read_done_resend_to_mds(struct nfs_pgio_header *hdr)
Trond Myklebust62e4a762011-11-10 14:30:37 -05002214{
2215 struct nfs_pageio_descriptor pgio;
2216
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002217 /* Resend all requests through the MDS */
Weston Andros Adamson53113ad2014-06-09 11:48:38 -04002218 nfs_pageio_init_read(&pgio, hdr->inode, true, hdr->completion_ops);
2219 return nfs_pageio_resend(&pgio, hdr);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002220}
Andy Adamsone7dd79af2012-04-27 17:53:46 -04002221EXPORT_SYMBOL_GPL(pnfs_read_done_resend_to_mds);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002222
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002223static void pnfs_ld_handle_read_error(struct nfs_pgio_header *hdr)
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002224{
Fred Isamancd841602012-04-20 14:47:44 -04002225 dprintk("pnfs read error = %d\n", hdr->pnfs_error);
2226 if (NFS_SERVER(hdr->inode)->pnfs_curr_ld->flags &
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002227 PNFS_LAYOUTRET_ON_ERROR) {
Fred Isamancd841602012-04-20 14:47:44 -04002228 pnfs_return_layout(hdr->inode);
Fred Isaman1acbbb4e12012-04-20 14:47:37 -04002229 }
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002230 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags))
Weston Andros Adamson53113ad2014-06-09 11:48:38 -04002231 hdr->task.tk_status = pnfs_read_done_resend_to_mds(hdr);
Trond Myklebust62e4a762011-11-10 14:30:37 -05002232}
2233
Andy Adamsonb1f69b72010-10-20 00:18:03 -04002234/*
Benny Halevyd20581a2011-05-22 19:52:03 +03002235 * Called by non rpc-based layout drivers
2236 */
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002237void pnfs_ld_read_done(struct nfs_pgio_header *hdr)
Benny Halevyd20581a2011-05-22 19:52:03 +03002238{
Trond Myklebustbfc505d2016-09-15 18:26:05 -04002239 if (likely(!hdr->pnfs_error))
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002240 hdr->mds_ops->rpc_call_done(&hdr->task, hdr);
Kinglong Meef8417b482015-10-16 17:23:29 +08002241 trace_nfs4_pnfs_read(hdr, hdr->pnfs_error);
2242 if (unlikely(hdr->pnfs_error))
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002243 pnfs_ld_handle_read_error(hdr);
2244 hdr->mds_ops->rpc_release(hdr);
Benny Halevyd20581a2011-05-22 19:52:03 +03002245}
2246EXPORT_SYMBOL_GPL(pnfs_ld_read_done);
2247
Trond Myklebust493292d2011-07-13 15:58:28 -04002248static void
2249pnfs_read_through_mds(struct nfs_pageio_descriptor *desc,
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002250 struct nfs_pgio_header *hdr)
Trond Myklebust493292d2011-07-13 15:58:28 -04002251{
Peng Tao48d635f2014-11-10 08:35:35 +08002252 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04002253
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002254 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04002255 list_splice_tail_init(&hdr->pages, &mirror->pg_list);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002256 nfs_pageio_reset_read_mds(desc);
Weston Andros Adamsona7d42dd2014-09-19 10:55:07 -04002257 mirror->pg_recoalesce = 1;
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002258 }
Trond Myklebust1ca018d2015-06-17 19:41:51 -04002259 hdr->release(hdr);
Trond Myklebust493292d2011-07-13 15:58:28 -04002260}
2261
Benny Halevyd20581a2011-05-22 19:52:03 +03002262/*
Andy Adamson64419a92011-03-01 01:34:16 +00002263 * Call the appropriate parallel I/O subsystem read function.
2264 */
Trond Myklebust493292d2011-07-13 15:58:28 -04002265static enum pnfs_try_status
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002266pnfs_try_to_read_data(struct nfs_pgio_header *hdr,
Trond Myklebust493292d2011-07-13 15:58:28 -04002267 const struct rpc_call_ops *call_ops,
2268 struct pnfs_layout_segment *lseg)
Andy Adamson64419a92011-03-01 01:34:16 +00002269{
Fred Isamancd841602012-04-20 14:47:44 -04002270 struct inode *inode = hdr->inode;
Andy Adamson64419a92011-03-01 01:34:16 +00002271 struct nfs_server *nfss = NFS_SERVER(inode);
2272 enum pnfs_try_status trypnfs;
2273
Fred Isamancd841602012-04-20 14:47:44 -04002274 hdr->mds_ops = call_ops;
Andy Adamson64419a92011-03-01 01:34:16 +00002275
2276 dprintk("%s: Reading ino:%lu %u@%llu\n",
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002277 __func__, inode->i_ino, hdr->args.count, hdr->args.offset);
Andy Adamson64419a92011-03-01 01:34:16 +00002278
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002279 trypnfs = nfss->pnfs_curr_ld->read_pagelist(hdr);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002280 if (trypnfs != PNFS_NOT_ATTEMPTED)
Andy Adamson64419a92011-03-01 01:34:16 +00002281 nfs_inc_stats(inode, NFSIOS_PNFS_READ);
Andy Adamson64419a92011-03-01 01:34:16 +00002282 dprintk("%s End (trypnfs:%d)\n", __func__, trypnfs);
2283 return trypnfs;
2284}
Andy Adamson863a3c62011-03-23 13:27:54 +00002285
Peng Taoceb11e12014-11-10 08:35:38 +08002286/* Resend all requests through pnfs. */
Weston Andros Adamson1b1bc662016-04-01 11:42:28 -04002287void pnfs_read_resend_pnfs(struct nfs_pgio_header *hdr)
Peng Taoceb11e12014-11-10 08:35:38 +08002288{
2289 struct nfs_pageio_descriptor pgio;
2290
Weston Andros Adamson1b1bc662016-04-01 11:42:28 -04002291 if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
Trond Myklebustba15def2016-11-27 15:12:39 -05002292 /* Prevent deadlocks with layoutreturn! */
2293 pnfs_put_lseg(hdr->lseg);
2294 hdr->lseg = NULL;
2295
Weston Andros Adamson1b1bc662016-04-01 11:42:28 -04002296 nfs_pageio_init_read(&pgio, hdr->inode, false,
2297 hdr->completion_ops);
2298 hdr->task.tk_status = nfs_pageio_resend(&pgio, hdr);
2299 }
Peng Taoceb11e12014-11-10 08:35:38 +08002300}
2301EXPORT_SYMBOL_GPL(pnfs_read_resend_pnfs);
2302
Trond Myklebust493292d2011-07-13 15:58:28 -04002303static void
Weston Andros Adamson7f714722014-05-15 11:56:53 -04002304pnfs_do_read(struct nfs_pageio_descriptor *desc, struct nfs_pgio_header *hdr)
Trond Myklebust493292d2011-07-13 15:58:28 -04002305{
Trond Myklebust493292d2011-07-13 15:58:28 -04002306 const struct rpc_call_ops *call_ops = desc->pg_rpc_callops;
2307 struct pnfs_layout_segment *lseg = desc->pg_lseg;
Weston Andros Adamson7f714722014-05-15 11:56:53 -04002308 enum pnfs_try_status trypnfs;
Trond Myklebust493292d2011-07-13 15:58:28 -04002309
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002310 trypnfs = pnfs_try_to_read_data(hdr, call_ops, lseg);
Peng Taoceb11e12014-11-10 08:35:38 +08002311 if (trypnfs == PNFS_TRY_AGAIN)
Weston Andros Adamson1b1bc662016-04-01 11:42:28 -04002312 pnfs_read_resend_pnfs(hdr);
2313 if (trypnfs == PNFS_NOT_ATTEMPTED || hdr->task.tk_status)
Weston Andros Adamsond45f60c2014-06-09 11:48:35 -04002314 pnfs_read_through_mds(desc, hdr);
Trond Myklebust493292d2011-07-13 15:58:28 -04002315}
2316
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002317static void pnfs_readhdr_free(struct nfs_pgio_header *hdr)
2318{
Trond Myklebust9369a432012-09-18 20:57:08 -04002319 pnfs_put_lseg(hdr->lseg);
Weston Andros Adamson1e7f3a42014-06-09 11:48:33 -04002320 nfs_pgio_header_free(hdr);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002321}
2322
Trond Myklebust493292d2011-07-13 15:58:28 -04002323int
2324pnfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
2325{
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002326 struct nfs_pgio_header *hdr;
Trond Myklebust493292d2011-07-13 15:58:28 -04002327 int ret;
2328
Weston Andros Adamson1e7f3a42014-06-09 11:48:33 -04002329 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
2330 if (!hdr) {
Peng Tao2bff2282015-12-05 02:03:17 +08002331 desc->pg_error = -ENOMEM;
2332 return desc->pg_error;
Trond Myklebust493292d2011-07-13 15:58:28 -04002333 }
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002334 nfs_pgheader_init(desc, hdr, pnfs_readhdr_free);
Trond Myklebust9369a432012-09-18 20:57:08 -04002335 hdr->lseg = pnfs_get_lseg(desc->pg_lseg);
Anna Schumakeref2c4882014-05-06 09:12:36 -04002336 ret = nfs_generic_pgio(desc, hdr);
Weston Andros Adamson180bb5e2014-09-10 15:48:01 -04002337 if (!ret)
Weston Andros Adamson7f714722014-05-15 11:56:53 -04002338 pnfs_do_read(desc, hdr);
Fred Isaman4db6e0b2012-04-20 14:47:46 -04002339 return ret;
Trond Myklebust493292d2011-07-13 15:58:28 -04002340}
2341EXPORT_SYMBOL_GPL(pnfs_generic_pg_readpages);
2342
Trond Myklebust71244d92014-01-13 13:34:36 -05002343static void pnfs_clear_layoutcommitting(struct inode *inode)
2344{
2345 unsigned long *bitlock = &NFS_I(inode)->flags;
2346
2347 clear_bit_unlock(NFS_INO_LAYOUTCOMMITTING, bitlock);
Peter Zijlstra4e857c52014-03-17 18:06:10 +01002348 smp_mb__after_atomic();
Trond Myklebust71244d92014-01-13 13:34:36 -05002349 wake_up_bit(bitlock, NFS_INO_LAYOUTCOMMITTING);
2350}
2351
Andy Adamson863a3c62011-03-23 13:27:54 +00002352/*
Peng Taoa9bae562011-07-30 20:52:33 -04002353 * There can be multiple RW segments.
Andy Adamson863a3c62011-03-23 13:27:54 +00002354 */
Peng Taoa9bae562011-07-30 20:52:33 -04002355static void pnfs_list_write_lseg(struct inode *inode, struct list_head *listp)
Andy Adamson863a3c62011-03-23 13:27:54 +00002356{
Peng Taoa9bae562011-07-30 20:52:33 -04002357 struct pnfs_layout_segment *lseg;
Andy Adamson863a3c62011-03-23 13:27:54 +00002358
Peng Taoa9bae562011-07-30 20:52:33 -04002359 list_for_each_entry(lseg, &NFS_I(inode)->layout->plh_segs, pls_list) {
2360 if (lseg->pls_range.iomode == IOMODE_RW &&
Trond Myklebusta073dbf2013-03-20 12:34:32 -04002361 test_and_clear_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags))
Peng Taoa9bae562011-07-30 20:52:33 -04002362 list_add(&lseg->pls_lc_list, listp);
2363 }
Andy Adamson863a3c62011-03-23 13:27:54 +00002364}
2365
Trond Myklebusta073dbf2013-03-20 12:34:32 -04002366static void pnfs_list_write_lseg_done(struct inode *inode, struct list_head *listp)
2367{
2368 struct pnfs_layout_segment *lseg, *tmp;
Trond Myklebusta073dbf2013-03-20 12:34:32 -04002369
2370 /* Matched by references in pnfs_set_layoutcommit */
2371 list_for_each_entry_safe(lseg, tmp, listp, pls_lc_list) {
2372 list_del_init(&lseg->pls_lc_list);
2373 pnfs_put_lseg(lseg);
2374 }
2375
Trond Myklebust71244d92014-01-13 13:34:36 -05002376 pnfs_clear_layoutcommitting(inode);
Trond Myklebusta073dbf2013-03-20 12:34:32 -04002377}
2378
Peng Tao1b0ae062011-09-22 21:50:12 -04002379void pnfs_set_lo_fail(struct pnfs_layout_segment *lseg)
2380{
Trond Myklebustb9e028f2012-09-18 16:41:18 -04002381 pnfs_layout_io_set_failed(lseg->pls_layout, lseg->pls_range.iomode);
Peng Tao1b0ae062011-09-22 21:50:12 -04002382}
2383EXPORT_SYMBOL_GPL(pnfs_set_lo_fail);
2384
Andy Adamson863a3c62011-03-23 13:27:54 +00002385void
Trond Myklebust67af7612015-03-25 20:40:38 -04002386pnfs_set_layoutcommit(struct inode *inode, struct pnfs_layout_segment *lseg,
2387 loff_t end_pos)
Andy Adamson863a3c62011-03-23 13:27:54 +00002388{
Fred Isamancd841602012-04-20 14:47:44 -04002389 struct nfs_inode *nfsi = NFS_I(inode);
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04002390 bool mark_as_dirty = false;
Andy Adamson863a3c62011-03-23 13:27:54 +00002391
Fred Isamancd841602012-04-20 14:47:44 -04002392 spin_lock(&inode->i_lock);
Andy Adamson863a3c62011-03-23 13:27:54 +00002393 if (!test_and_set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags)) {
Trond Myklebust29559b12015-03-25 20:10:20 -04002394 nfsi->layout->plh_lwb = end_pos;
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04002395 mark_as_dirty = true;
Andy Adamson863a3c62011-03-23 13:27:54 +00002396 dprintk("%s: Set layoutcommit for inode %lu ",
Fred Isamancd841602012-04-20 14:47:44 -04002397 __func__, inode->i_ino);
Trond Myklebust29559b12015-03-25 20:10:20 -04002398 } else if (end_pos > nfsi->layout->plh_lwb)
2399 nfsi->layout->plh_lwb = end_pos;
Trond Myklebust67af7612015-03-25 20:40:38 -04002400 if (!test_and_set_bit(NFS_LSEG_LAYOUTCOMMIT, &lseg->pls_flags)) {
Peng Taoa9bae562011-07-30 20:52:33 -04002401 /* references matched in nfs4_layoutcommit_release */
Trond Myklebust67af7612015-03-25 20:40:38 -04002402 pnfs_get_lseg(lseg);
Peng Taoa9bae562011-07-30 20:52:33 -04002403 }
Fred Isamancd841602012-04-20 14:47:44 -04002404 spin_unlock(&inode->i_lock);
Peng Taoacff58802011-07-30 20:52:31 -04002405 dprintk("%s: lseg %p end_pos %llu\n",
Trond Myklebust67af7612015-03-25 20:40:38 -04002406 __func__, lseg, nfsi->layout->plh_lwb);
Weston Andros Adamson79a48a12011-04-13 10:53:51 -04002407
2408 /* if pnfs_layoutcommit_inode() runs between inode locks, the next one
2409 * will be a noop because NFS_INO_LAYOUTCOMMIT will not be set */
2410 if (mark_as_dirty)
Fred Isamancd841602012-04-20 14:47:44 -04002411 mark_inode_dirty_sync(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00002412}
2413EXPORT_SYMBOL_GPL(pnfs_set_layoutcommit);
2414
Andy Adamsondb29c082011-07-30 20:52:38 -04002415void pnfs_cleanup_layoutcommit(struct nfs4_layoutcommit_data *data)
2416{
2417 struct nfs_server *nfss = NFS_SERVER(data->args.inode);
2418
2419 if (nfss->pnfs_curr_ld->cleanup_layoutcommit)
2420 nfss->pnfs_curr_ld->cleanup_layoutcommit(data);
Trond Myklebusta073dbf2013-03-20 12:34:32 -04002421 pnfs_list_write_lseg_done(data->args.inode, &data->lseg_list);
Andy Adamsondb29c082011-07-30 20:52:38 -04002422}
2423
Andy Adamsonde4b15c2011-03-12 02:58:09 -05002424/*
2425 * For the LAYOUT4_NFSV4_1_FILES layout type, NFS_DATA_SYNC WRITEs and
2426 * NFS_UNSTABLE WRITEs with a COMMIT to data servers must store enough
2427 * data to disk to allow the server to recover the data if it crashes.
2428 * LAYOUTCOMMIT is only needed when the NFL4_UFLG_COMMIT_THRU_MDS flag
2429 * is off, and a COMMIT is sent to a data server, or
2430 * if WRITEs to a data server return NFS_DATA_SYNC.
2431 */
Andy Adamson863a3c62011-03-23 13:27:54 +00002432int
Andy Adamsonef311532011-03-12 02:58:10 -05002433pnfs_layoutcommit_inode(struct inode *inode, bool sync)
Andy Adamson863a3c62011-03-23 13:27:54 +00002434{
Christoph Hellwig5f919c92014-08-21 11:09:25 -05002435 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
Andy Adamson863a3c62011-03-23 13:27:54 +00002436 struct nfs4_layoutcommit_data *data;
2437 struct nfs_inode *nfsi = NFS_I(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00002438 loff_t end_pos;
Trond Myklebust71244d92014-01-13 13:34:36 -05002439 int status;
2440
2441 if (!pnfs_layoutcommit_outstanding(inode))
2442 return 0;
Andy Adamson863a3c62011-03-23 13:27:54 +00002443
2444 dprintk("--> %s inode %lu\n", __func__, inode->i_ino);
2445
Trond Myklebust71244d92014-01-13 13:34:36 -05002446 status = -EAGAIN;
2447 if (test_and_set_bit(NFS_INO_LAYOUTCOMMITTING, &nfsi->flags)) {
2448 if (!sync)
2449 goto out;
NeilBrown74316202014-07-07 15:16:04 +10002450 status = wait_on_bit_lock_action(&nfsi->flags,
Trond Myklebust71244d92014-01-13 13:34:36 -05002451 NFS_INO_LAYOUTCOMMITTING,
2452 nfs_wait_bit_killable,
2453 TASK_KILLABLE);
2454 if (status)
2455 goto out;
2456 }
Andy Adamsonde4b15c2011-03-12 02:58:09 -05002457
Trond Myklebust71244d92014-01-13 13:34:36 -05002458 status = -ENOMEM;
Andy Adamson863a3c62011-03-23 13:27:54 +00002459 /* Note kzalloc ensures data->res.seq_res.sr_slot == NULL */
2460 data = kzalloc(sizeof(*data), GFP_NOFS);
Trond Myklebust71244d92014-01-13 13:34:36 -05002461 if (!data)
2462 goto clear_layoutcommitting;
Andy Adamson863a3c62011-03-23 13:27:54 +00002463
Trond Myklebust71244d92014-01-13 13:34:36 -05002464 status = 0;
2465 spin_lock(&inode->i_lock);
2466 if (!test_and_clear_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags))
2467 goto out_unlock;
Peng Tao92407e72011-10-23 20:21:17 -07002468
Peng Taoa9bae562011-07-30 20:52:33 -04002469 INIT_LIST_HEAD(&data->lseg_list);
Peng Taoa9bae562011-07-30 20:52:33 -04002470 pnfs_list_write_lseg(inode, &data->lseg_list);
Andy Adamson863a3c62011-03-23 13:27:54 +00002471
Peng Taoacff58802011-07-30 20:52:31 -04002472 end_pos = nfsi->layout->plh_lwb;
Andy Adamson863a3c62011-03-23 13:27:54 +00002473
Trond Myklebustf597c532012-03-04 18:13:56 -05002474 nfs4_stateid_copy(&data->args.stateid, &nfsi->layout->plh_stateid);
Andy Adamson863a3c62011-03-23 13:27:54 +00002475 spin_unlock(&inode->i_lock);
2476
2477 data->args.inode = inode;
Peng Tao9fa40752011-07-30 20:52:32 -04002478 data->cred = get_rpccred(nfsi->layout->plh_lc_cred);
Andy Adamson863a3c62011-03-23 13:27:54 +00002479 nfs_fattr_init(&data->fattr);
2480 data->args.bitmask = NFS_SERVER(inode)->cache_consistency_bitmask;
2481 data->res.fattr = &data->fattr;
Trond Myklebust2e18d4d2016-06-26 18:54:58 -04002482 if (end_pos != 0)
2483 data->args.lastbytewritten = end_pos - 1;
2484 else
2485 data->args.lastbytewritten = U64_MAX;
Andy Adamson863a3c62011-03-23 13:27:54 +00002486 data->res.server = NFS_SERVER(inode);
2487
Christoph Hellwig5f919c92014-08-21 11:09:25 -05002488 if (ld->prepare_layoutcommit) {
2489 status = ld->prepare_layoutcommit(&data->args);
2490 if (status) {
Jeff Layton3471648a2015-07-10 15:58:42 -04002491 put_rpccred(data->cred);
Christoph Hellwig5f919c92014-08-21 11:09:25 -05002492 spin_lock(&inode->i_lock);
Trond Myklebust29559b12015-03-25 20:10:20 -04002493 set_bit(NFS_INO_LAYOUTCOMMIT, &nfsi->flags);
2494 if (end_pos > nfsi->layout->plh_lwb)
Christoph Hellwig5f919c92014-08-21 11:09:25 -05002495 nfsi->layout->plh_lwb = end_pos;
Jeff Layton3471648a2015-07-10 15:58:42 -04002496 goto out_unlock;
Christoph Hellwig5f919c92014-08-21 11:09:25 -05002497 }
2498 }
2499
2500
Andy Adamson863a3c62011-03-23 13:27:54 +00002501 status = nfs4_proc_layoutcommit(data, sync);
2502out:
Peng Tao92407e72011-10-23 20:21:17 -07002503 if (status)
2504 mark_inode_dirty_sync(inode);
Andy Adamson863a3c62011-03-23 13:27:54 +00002505 dprintk("<-- %s status %d\n", __func__, status);
2506 return status;
Trond Myklebust71244d92014-01-13 13:34:36 -05002507out_unlock:
2508 spin_unlock(&inode->i_lock);
Peng Tao92407e72011-10-23 20:21:17 -07002509 kfree(data);
Trond Myklebust71244d92014-01-13 13:34:36 -05002510clear_layoutcommitting:
2511 pnfs_clear_layoutcommitting(inode);
Peng Tao92407e72011-10-23 20:21:17 -07002512 goto out;
Andy Adamson863a3c62011-03-23 13:27:54 +00002513}
Peng Tao72cff442014-08-07 10:12:38 +08002514EXPORT_SYMBOL_GPL(pnfs_layoutcommit_inode);
Andy Adamson82be4172012-05-23 05:02:35 -04002515
Trond Myklebust5bb89b42015-03-25 14:14:42 -04002516int
2517pnfs_generic_sync(struct inode *inode, bool datasync)
2518{
2519 return pnfs_layoutcommit_inode(inode, true);
2520}
2521EXPORT_SYMBOL_GPL(pnfs_generic_sync);
2522
Andy Adamson82be4172012-05-23 05:02:35 -04002523struct nfs4_threshold *pnfs_mdsthreshold_alloc(void)
2524{
2525 struct nfs4_threshold *thp;
2526
2527 thp = kzalloc(sizeof(*thp), GFP_NOFS);
2528 if (!thp) {
2529 dprintk("%s mdsthreshold allocation failed\n", __func__);
2530 return NULL;
2531 }
2532 return thp;
2533}
Peng Tao87334082015-06-23 19:51:57 +08002534
Peng Tao865a7ec2015-06-25 18:19:32 +08002535#if IS_ENABLED(CONFIG_NFS_V4_2)
Peng Tao87334082015-06-23 19:51:57 +08002536int
Trond Myklebustc8ad8892015-08-05 17:31:58 -04002537pnfs_report_layoutstat(struct inode *inode, gfp_t gfp_flags)
Peng Tao87334082015-06-23 19:51:57 +08002538{
2539 struct pnfs_layoutdriver_type *ld = NFS_SERVER(inode)->pnfs_curr_ld;
2540 struct nfs_server *server = NFS_SERVER(inode);
Peng Tao1bfe3b22015-06-23 19:52:03 +08002541 struct nfs_inode *nfsi = NFS_I(inode);
Peng Tao87334082015-06-23 19:51:57 +08002542 struct nfs42_layoutstat_data *data;
2543 struct pnfs_layout_hdr *hdr;
2544 int status = 0;
2545
2546 if (!pnfs_enabled_sb(server) || !ld->prepare_layoutstats)
2547 goto out;
2548
Trond Myklebust6c5a0d82015-06-27 11:45:46 -04002549 if (!nfs_server_capable(inode, NFS_CAP_LAYOUTSTATS))
2550 goto out;
2551
Peng Tao1bfe3b22015-06-23 19:52:03 +08002552 if (test_and_set_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags))
2553 goto out;
2554
Peng Tao87334082015-06-23 19:51:57 +08002555 spin_lock(&inode->i_lock);
2556 if (!NFS_I(inode)->layout) {
2557 spin_unlock(&inode->i_lock);
Trond Myklebustf538d0b2016-05-16 14:41:14 -04002558 goto out_clear_layoutstats;
Peng Tao87334082015-06-23 19:51:57 +08002559 }
2560 hdr = NFS_I(inode)->layout;
2561 pnfs_get_layout_hdr(hdr);
2562 spin_unlock(&inode->i_lock);
2563
Trond Myklebustc8ad8892015-08-05 17:31:58 -04002564 data = kzalloc(sizeof(*data), gfp_flags);
Peng Tao87334082015-06-23 19:51:57 +08002565 if (!data) {
2566 status = -ENOMEM;
2567 goto out_put;
2568 }
2569
2570 data->args.fh = NFS_FH(inode);
2571 data->args.inode = inode;
Peng Tao87334082015-06-23 19:51:57 +08002572 status = ld->prepare_layoutstats(&data->args);
2573 if (status)
2574 goto out_free;
2575
2576 status = nfs42_proc_layoutstats_generic(NFS_SERVER(inode), data);
2577
2578out:
2579 dprintk("%s returns %d\n", __func__, status);
2580 return status;
2581
2582out_free:
2583 kfree(data);
2584out_put:
2585 pnfs_put_layout_hdr(hdr);
Trond Myklebustf538d0b2016-05-16 14:41:14 -04002586out_clear_layoutstats:
Peng Tao1bfe3b22015-06-23 19:52:03 +08002587 smp_mb__before_atomic();
2588 clear_bit(NFS_INO_LAYOUTSTATS, &nfsi->flags);
2589 smp_mb__after_atomic();
Peng Tao87334082015-06-23 19:51:57 +08002590 goto out;
2591}
2592EXPORT_SYMBOL_GPL(pnfs_report_layoutstat);
Peng Tao865a7ec2015-06-25 18:19:32 +08002593#endif
Trond Myklebustbbf58bf2015-08-24 20:39:18 -04002594
2595unsigned int layoutstats_timer;
2596module_param(layoutstats_timer, uint, 0644);
2597EXPORT_SYMBOL_GPL(layoutstats_timer);