blob: 1ab91124a93e9f380b95dd7eea754dbd8864f619 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/nfs/direct.c
3 *
4 * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
5 *
6 * High-performance uncached I/O for the Linux NFS client
7 *
8 * There are important applications whose performance or correctness
9 * depends on uncached access to file data. Database clusters
Chuck Lever88467052006-03-20 13:44:34 -050010 * (multiple copies of the same instance running on separate hosts)
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * implement their own cache coherency protocol that subsumes file
Chuck Lever88467052006-03-20 13:44:34 -050012 * system cache protocols. Applications that process datasets
13 * considerably larger than the client's memory do not always benefit
14 * from a local cache. A streaming video server, for instance, has no
Linus Torvalds1da177e2005-04-16 15:20:36 -070015 * need to cache the contents of a file.
16 *
17 * When an application requests uncached I/O, all read and write requests
18 * are made directly to the server; data stored or fetched via these
19 * requests is not cached in the Linux page cache. The client does not
20 * correct unaligned requests from applications. All requested bytes are
21 * held on permanent storage before a direct write system call returns to
22 * an application.
23 *
24 * Solaris implements an uncached I/O facility called directio() that
25 * is used for backups and sequential I/O to very large files. Solaris
26 * also supports uncaching whole NFS partitions with "-o forcedirectio,"
27 * an undocumented mount option.
28 *
29 * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
30 * help from Andrew Morton.
31 *
32 * 18 Dec 2001 Initial implementation for 2.4 --cel
33 * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy
34 * 08 Jun 2003 Port to 2.5 APIs --cel
35 * 31 Mar 2004 Handle direct I/O without VFS support --cel
36 * 15 Sep 2004 Parallel async reads --cel
Chuck Lever88467052006-03-20 13:44:34 -050037 * 04 May 2005 support O_DIRECT with aio --cel
Linus Torvalds1da177e2005-04-16 15:20:36 -070038 *
39 */
40
Linus Torvalds1da177e2005-04-16 15:20:36 -070041#include <linux/errno.h>
42#include <linux/sched.h>
43#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070044#include <linux/file.h>
45#include <linux/pagemap.h>
46#include <linux/kref.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090047#include <linux/slab.h>
Konstantin Khlebnikov7ec10f22011-02-22 00:28:34 +030048#include <linux/task_io_accounting_ops.h>
Peng Tao62965562012-09-25 14:55:57 +080049#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070050
51#include <linux/nfs_fs.h>
52#include <linux/nfs_page.h>
53#include <linux/sunrpc/clnt.h>
54
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#include <asm/uaccess.h>
Arun Sharma600634972011-07-26 16:09:06 -070056#include <linux/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070057
Trond Myklebust8d5658c2007-04-10 09:26:35 -040058#include "internal.h"
Chuck Lever91d5b472006-03-20 13:44:14 -050059#include "iostat.h"
Fred Isaman1763da12012-04-20 14:47:57 -040060#include "pnfs.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070061
Linus Torvalds1da177e2005-04-16 15:20:36 -070062#define NFSDBG_FACILITY NFSDBG_VFS
Linus Torvalds1da177e2005-04-16 15:20:36 -070063
Christoph Lametere18b8902006-12-06 20:33:20 -080064static struct kmem_cache *nfs_direct_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -070065
66/*
67 * This represents a set of asynchronous requests that we're waiting on
68 */
Weston Andros Adamson0a00b772014-09-19 12:48:33 -040069struct nfs_direct_mirror {
70 ssize_t count;
71};
72
Linus Torvalds1da177e2005-04-16 15:20:36 -070073struct nfs_direct_req {
74 struct kref kref; /* release manager */
Chuck Lever15ce4a02006-03-20 13:44:34 -050075
76 /* I/O parameters */
Trond Myklebusta8881f52006-03-20 13:44:36 -050077 struct nfs_open_context *ctx; /* file open context info */
Trond Myklebustf11ac8d2010-06-25 16:35:53 -040078 struct nfs_lock_context *l_ctx; /* Lock context info */
Chuck Lever99514f82006-03-20 13:44:30 -050079 struct kiocb * iocb; /* controlling i/o request */
Chuck Lever88467052006-03-20 13:44:34 -050080 struct inode * inode; /* target file of i/o */
Chuck Lever15ce4a02006-03-20 13:44:34 -050081
82 /* completion state */
Trond Myklebust607f31e2006-06-28 16:52:45 -040083 atomic_t io_count; /* i/os we're waiting for */
Chuck Lever15ce4a02006-03-20 13:44:34 -050084 spinlock_t lock; /* protect completion state */
Weston Andros Adamson0a00b772014-09-19 12:48:33 -040085
86 struct nfs_direct_mirror mirrors[NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX];
87 int mirror_count;
88
Trond Myklebustfb1f4102018-03-06 12:47:08 -050089 loff_t io_start; /* Start offset for I/O */
Chuck Lever15ce4a02006-03-20 13:44:34 -050090 ssize_t count, /* bytes actually processed */
Weston Andros Adamsoned3743a2016-04-01 11:42:29 -040091 max_count, /* max expected count */
Peng Tao35754bc2012-09-25 14:55:57 +080092 bytes_left, /* bytes left to be sent */
Linus Torvalds1da177e2005-04-16 15:20:36 -070093 error; /* any reported error */
Trond Myklebustd72b7a62006-03-20 13:44:43 -050094 struct completion completion; /* wait for i/o completion */
Trond Myklebustfad61492006-03-20 13:44:36 -050095
96 /* commit state */
Fred Isaman1763da12012-04-20 14:47:57 -040097 struct nfs_mds_commit_info mds_cinfo; /* Storage for cinfo */
98 struct pnfs_ds_commit_info ds_cinfo; /* Storage for cinfo */
99 struct work_struct work;
Trond Myklebustfad61492006-03-20 13:44:36 -0500100 int flags;
101#define NFS_ODIRECT_DO_COMMIT (1) /* an unstable reply was received */
102#define NFS_ODIRECT_RESCHED_WRITES (2) /* write verification failed */
103 struct nfs_writeverf verf; /* unstable write verifier */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700104};
105
Fred Isaman1763da12012-04-20 14:47:57 -0400106static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops;
107static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops;
Trond Myklebustfad61492006-03-20 13:44:36 -0500108static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode);
Fred Isaman1763da12012-04-20 14:47:57 -0400109static void nfs_direct_write_schedule_work(struct work_struct *work);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400110
111static inline void get_dreq(struct nfs_direct_req *dreq)
112{
113 atomic_inc(&dreq->io_count);
114}
115
116static inline int put_dreq(struct nfs_direct_req *dreq)
117{
118 return atomic_dec_and_test(&dreq->io_count);
119}
120
Weston Andros Adamson0a00b772014-09-19 12:48:33 -0400121static void
122nfs_direct_good_bytes(struct nfs_direct_req *dreq, struct nfs_pgio_header *hdr)
123{
124 int i;
125 ssize_t count;
126
Weston Andros Adamsoned3743a2016-04-01 11:42:29 -0400127 WARN_ON_ONCE(dreq->count >= dreq->max_count);
128
Peng Tao1ccbad92015-04-09 23:02:16 +0800129 if (dreq->mirror_count == 1) {
130 dreq->mirrors[hdr->pgio_mirror_idx].count += hdr->good_bytes;
131 dreq->count += hdr->good_bytes;
132 } else {
133 /* mirrored writes */
134 count = dreq->mirrors[hdr->pgio_mirror_idx].count;
135 if (count + dreq->io_start < hdr->io_start + hdr->good_bytes) {
136 count = hdr->io_start + hdr->good_bytes - dreq->io_start;
137 dreq->mirrors[hdr->pgio_mirror_idx].count = count;
138 }
139 /* update the dreq->count by finding the minimum agreed count from all
140 * mirrors */
141 count = dreq->mirrors[0].count;
Weston Andros Adamson0a00b772014-09-19 12:48:33 -0400142
Peng Tao1ccbad92015-04-09 23:02:16 +0800143 for (i = 1; i < dreq->mirror_count; i++)
144 count = min(count, dreq->mirrors[i].count);
145
146 dreq->count = count;
Peng Tao5fadeb42015-01-19 12:41:16 +0800147 }
Weston Andros Adamson0a00b772014-09-19 12:48:33 -0400148}
149
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400150/*
151 * nfs_direct_select_verf - select the right verifier
152 * @dreq - direct request possibly spanning multiple servers
153 * @ds_clp - nfs_client of data server or NULL if MDS / non-pnfs
Weston Andros Adamson6cccbb62014-09-16 17:35:51 -0400154 * @commit_idx - commit bucket index for the DS
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400155 *
156 * returns the correct verifier to use given the role of the server
157 */
158static struct nfs_writeverf *
159nfs_direct_select_verf(struct nfs_direct_req *dreq,
160 struct nfs_client *ds_clp,
Weston Andros Adamson6cccbb62014-09-16 17:35:51 -0400161 int commit_idx)
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400162{
163 struct nfs_writeverf *verfp = &dreq->verf;
164
165#ifdef CONFIG_NFS_V4_1
Kinglong Mee834e4652015-09-22 06:54:47 +0800166 /*
167 * pNFS is in use, use the DS verf except commit_through_mds is set
168 * for layout segment where nbuckets is zero.
169 */
170 if (ds_clp && dreq->ds_cinfo.nbuckets > 0) {
Weston Andros Adamson6cccbb62014-09-16 17:35:51 -0400171 if (commit_idx >= 0 && commit_idx < dreq->ds_cinfo.nbuckets)
172 verfp = &dreq->ds_cinfo.buckets[commit_idx].direct_verf;
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400173 else
174 WARN_ON_ONCE(1);
175 }
176#endif
177 return verfp;
178}
179
180
181/*
182 * nfs_direct_set_hdr_verf - set the write/commit verifier
183 * @dreq - direct request possibly spanning multiple servers
184 * @hdr - pageio header to validate against previously seen verfs
185 *
186 * Set the server's (MDS or DS) "seen" verifier
187 */
188static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq,
189 struct nfs_pgio_header *hdr)
190{
191 struct nfs_writeverf *verfp;
192
Weston Andros Adamson6cccbb62014-09-16 17:35:51 -0400193 verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400194 WARN_ON_ONCE(verfp->committed >= 0);
195 memcpy(verfp, &hdr->verf, sizeof(struct nfs_writeverf));
196 WARN_ON_ONCE(verfp->committed < 0);
197}
198
Trond Myklebust8fc3c382016-06-01 21:32:24 -0400199static int nfs_direct_cmp_verf(const struct nfs_writeverf *v1,
200 const struct nfs_writeverf *v2)
201{
202 return nfs_write_verifier_cmp(&v1->verifier, &v2->verifier);
203}
204
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400205/*
206 * nfs_direct_cmp_hdr_verf - compare verifier for pgio header
207 * @dreq - direct request possibly spanning multiple servers
208 * @hdr - pageio header to validate against previously seen verf
209 *
210 * set the server's "seen" verf if not initialized.
211 * returns result of comparison between @hdr->verf and the "seen"
212 * verf of the server used by @hdr (DS or MDS)
213 */
214static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq,
215 struct nfs_pgio_header *hdr)
216{
217 struct nfs_writeverf *verfp;
218
Weston Andros Adamson6cccbb62014-09-16 17:35:51 -0400219 verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400220 if (verfp->committed < 0) {
221 nfs_direct_set_hdr_verf(dreq, hdr);
222 return 0;
223 }
Trond Myklebust8fc3c382016-06-01 21:32:24 -0400224 return nfs_direct_cmp_verf(verfp, &hdr->verf);
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400225}
226
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400227/*
228 * nfs_direct_cmp_commit_data_verf - compare verifier for commit data
229 * @dreq - direct request possibly spanning multiple servers
230 * @data - commit data to validate against previously seen verf
231 *
232 * returns result of comparison between @data->verf and the verf of
233 * the server used by @data (DS or MDS)
234 */
235static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq,
236 struct nfs_commit_data *data)
237{
238 struct nfs_writeverf *verfp;
239
240 verfp = nfs_direct_select_verf(dreq, data->ds_clp,
241 data->ds_commit_index);
Weston Andros Adamson80c76fe2014-10-01 12:58:25 -0400242
243 /* verifier not set so always fail */
244 if (verfp->committed < 0)
245 return 1;
246
Trond Myklebust8fc3c382016-06-01 21:32:24 -0400247 return nfs_direct_cmp_verf(verfp, &data->verf);
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400248}
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400249
Linus Torvalds1da177e2005-04-16 15:20:36 -0700250/**
Chuck Leverb8a32e22006-03-20 13:44:28 -0500251 * nfs_direct_IO - NFS address space operation for direct I/O
Chuck Leverb8a32e22006-03-20 13:44:28 -0500252 * @iocb: target I/O control block
Al Viro90090ae2016-05-29 19:05:03 -0400253 * @iter: I/O buffer
Chuck Leverb8a32e22006-03-20 13:44:28 -0500254 *
255 * The presence of this routine in the address space ops vector means
Mel Gormana564b8f2012-07-31 16:45:12 -0700256 * the NFS client supports direct I/O. However, for most direct IO, we
257 * shunt off direct read and write requests before the VFS gets them,
258 * so this method is only ever called for swap.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700259 */
Christoph Hellwigc8b8e322016-04-07 08:51:58 -0700260ssize_t nfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
Chuck Leverb8a32e22006-03-20 13:44:28 -0500261{
Peng Taoee8a1a82015-01-20 07:44:29 +0800262 struct inode *inode = iocb->ki_filp->f_mapping->host;
263
264 /* we only support swap file calling nfs_direct_IO */
265 if (!IS_SWAPFILE(inode))
266 return 0;
267
Christoph Hellwig66ee59a2015-02-11 19:56:46 +0100268 VM_BUG_ON(iov_iter_count(iter) != PAGE_SIZE);
Mel Gormana564b8f2012-07-31 16:45:12 -0700269
Omar Sandoval6f673762015-03-16 04:33:52 -0700270 if (iov_iter_rw(iter) == READ)
Christoph Hellwigc8b8e322016-04-07 08:51:58 -0700271 return nfs_file_direct_read(iocb, iter);
Al Viro65a4a1c2015-04-09 14:11:08 -0400272 return nfs_file_direct_write(iocb, iter);
Chuck Leverb8a32e22006-03-20 13:44:28 -0500273}
274
Chuck Lever749e1462007-05-19 17:22:46 -0400275static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
Chuck Lever9c93ab72006-06-20 12:56:31 -0400276{
Chuck Lever749e1462007-05-19 17:22:46 -0400277 unsigned int i;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400278 for (i = 0; i < npages; i++)
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300279 put_page(pages[i]);
Trond Myklebust6b45d852006-03-20 13:44:43 -0500280}
281
Fred Isaman1763da12012-04-20 14:47:57 -0400282void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo,
283 struct nfs_direct_req *dreq)
284{
Dave Wysochanskife238e62016-04-01 13:45:09 -0400285 cinfo->inode = dreq->inode;
Fred Isaman1763da12012-04-20 14:47:57 -0400286 cinfo->mds = &dreq->mds_cinfo;
287 cinfo->ds = &dreq->ds_cinfo;
288 cinfo->dreq = dreq;
289 cinfo->completion_ops = &nfs_direct_commit_completion_ops;
290}
291
Weston Andros Adamson0a00b772014-09-19 12:48:33 -0400292static inline void nfs_direct_setup_mirroring(struct nfs_direct_req *dreq,
293 struct nfs_pageio_descriptor *pgio,
294 struct nfs_page *req)
295{
296 int mirror_count = 1;
297
298 if (pgio->pg_ops->pg_get_mirror_count)
299 mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);
300
301 dreq->mirror_count = mirror_count;
302}
303
Chuck Lever93619e52006-03-20 13:44:31 -0500304static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700305{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306 struct nfs_direct_req *dreq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307
Trond Myklebust292f3ee2012-04-30 18:31:49 -0400308 dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700309 if (!dreq)
310 return NULL;
311
312 kref_init(&dreq->kref);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400313 kref_get(&dreq->kref);
Trond Myklebustd72b7a62006-03-20 13:44:43 -0500314 init_completion(&dreq->completion);
Fred Isaman1763da12012-04-20 14:47:57 -0400315 INIT_LIST_HEAD(&dreq->mds_cinfo.list);
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400316 dreq->verf.committed = NFS_INVALID_STABLE_HOW; /* not set yet */
Fred Isaman1763da12012-04-20 14:47:57 -0400317 INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);
Weston Andros Adamson0a00b772014-09-19 12:48:33 -0400318 dreq->mirror_count = 1;
Chuck Lever15ce4a02006-03-20 13:44:34 -0500319 spin_lock_init(&dreq->lock);
Chuck Lever93619e52006-03-20 13:44:31 -0500320
321 return dreq;
322}
323
Trond Myklebustb4946ff2007-05-30 12:58:00 -0400324static void nfs_direct_req_free(struct kref *kref)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325{
326 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
Trond Myklebusta8881f52006-03-20 13:44:36 -0500327
Peng Tao8c393f92014-11-05 22:36:50 +0800328 nfs_free_pnfs_ds_cinfo(&dreq->ds_cinfo);
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400329 if (dreq->l_ctx != NULL)
330 nfs_put_lock_context(dreq->l_ctx);
Trond Myklebusta8881f52006-03-20 13:44:36 -0500331 if (dreq->ctx != NULL)
332 put_nfs_open_context(dreq->ctx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700333 kmem_cache_free(nfs_direct_cachep, dreq);
334}
335
Trond Myklebustb4946ff2007-05-30 12:58:00 -0400336static void nfs_direct_req_release(struct nfs_direct_req *dreq)
337{
338 kref_put(&dreq->kref, nfs_direct_req_free);
339}
340
Peng Tao62965562012-09-25 14:55:57 +0800341ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq)
342{
343 return dreq->bytes_left;
344}
345EXPORT_SYMBOL_GPL(nfs_dreq_bytes_left);
346
Chuck Leverd4cc9482006-03-20 13:44:28 -0500347/*
Chuck Leverbc0fb202006-03-20 13:44:31 -0500348 * Collects and returns the final error value/byte-count.
349 */
350static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
351{
Chuck Lever15ce4a02006-03-20 13:44:34 -0500352 ssize_t result = -EIOCBQUEUED;
Chuck Leverbc0fb202006-03-20 13:44:31 -0500353
354 /* Async requests don't wait here */
355 if (dreq->iocb)
356 goto out;
357
Matthew Wilcox150030b2007-12-06 16:24:39 -0500358 result = wait_for_completion_killable(&dreq->completion);
Chuck Leverbc0fb202006-03-20 13:44:31 -0500359
Trond Myklebustd2a7de02016-06-17 16:48:22 -0400360 if (!result) {
361 result = dreq->count;
362 WARN_ON_ONCE(dreq->count < 0);
363 }
Chuck Leverbc0fb202006-03-20 13:44:31 -0500364 if (!result)
Chuck Lever15ce4a02006-03-20 13:44:34 -0500365 result = dreq->error;
Chuck Leverbc0fb202006-03-20 13:44:31 -0500366
367out:
Chuck Leverbc0fb202006-03-20 13:44:31 -0500368 return (ssize_t) result;
369}
370
371/*
Trond Myklebust607f31e2006-06-28 16:52:45 -0400372 * Synchronous I/O uses a stack-allocated iocb. Thus we can't trust
373 * the iocb is still valid here if this is a synchronous request.
Chuck Lever63ab46a2006-03-20 13:44:31 -0500374 */
Trond Myklebustf7b5c342016-06-23 09:29:47 -0400375static void nfs_direct_complete(struct nfs_direct_req *dreq)
Chuck Lever63ab46a2006-03-20 13:44:31 -0500376{
Christoph Hellwig9811cd52013-11-14 08:50:28 -0800377 struct inode *inode = dreq->inode;
378
Jens Axboefe0f07d2015-04-15 17:05:48 -0600379 inode_dio_end(inode);
Christoph Hellwig2a009ec2013-11-14 08:50:29 -0800380
381 if (dreq->iocb) {
Chuck Lever15ce4a02006-03-20 13:44:34 -0500382 long res = (long) dreq->error;
Trond Myklebustd2a7de02016-06-17 16:48:22 -0400383 if (dreq->count != 0) {
Chuck Lever15ce4a02006-03-20 13:44:34 -0500384 res = (long) dreq->count;
Trond Myklebustd2a7de02016-06-17 16:48:22 -0400385 WARN_ON_ONCE(dreq->count < 0);
386 }
Christoph Hellwig04b2fa92015-02-02 14:49:06 +0100387 dreq->iocb->ki_complete(dreq->iocb, res, 0);
Trond Myklebustd72b7a62006-03-20 13:44:43 -0500388 }
Christoph Hellwig2a009ec2013-11-14 08:50:29 -0800389
Daniel Wagner024de8f2016-09-22 13:54:28 +0200390 complete(&dreq->completion);
Chuck Lever63ab46a2006-03-20 13:44:31 -0500391
Trond Myklebustb4946ff2007-05-30 12:58:00 -0400392 nfs_direct_req_release(dreq);
Chuck Lever63ab46a2006-03-20 13:44:31 -0500393}
394
Trond Myklebust1385b812012-05-04 13:54:24 -0400395static void nfs_direct_readpage_release(struct nfs_page *req)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396{
Niels de Vos1e8968c2013-12-17 18:20:16 +0100397 dprintk("NFS: direct read done (%s/%llu %d@%lld)\n",
Al Virofc640052016-04-10 01:33:30 -0400398 req->wb_context->dentry->d_sb->s_id,
David Howells2b0143b2015-03-17 22:25:59 +0000399 (unsigned long long)NFS_FILEID(d_inode(req->wb_context->dentry)),
Fred Isaman584aa812012-04-20 14:47:51 -0400400 req->wb_bytes,
401 (long long)req_offset(req));
402 nfs_release_request(req);
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400403}
404
Fred Isaman584aa812012-04-20 14:47:51 -0400405static void nfs_direct_read_completion(struct nfs_pgio_header *hdr)
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400406{
Fred Isaman584aa812012-04-20 14:47:51 -0400407 unsigned long bytes = 0;
408 struct nfs_direct_req *dreq = hdr->dreq;
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400409
Fred Isaman584aa812012-04-20 14:47:51 -0400410 if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
411 goto out_put;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700412
Chuck Lever15ce4a02006-03-20 13:44:34 -0500413 spin_lock(&dreq->lock);
Fred Isaman584aa812012-04-20 14:47:51 -0400414 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && (hdr->good_bytes == 0))
415 dreq->error = hdr->error;
416 else
Weston Andros Adamson0a00b772014-09-19 12:48:33 -0400417 nfs_direct_good_bytes(dreq, hdr);
418
Fred Isaman584aa812012-04-20 14:47:51 -0400419 spin_unlock(&dreq->lock);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400420
Trond Myklebust4bd8b012012-05-01 12:49:58 -0400421 while (!list_empty(&hdr->pages)) {
422 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
423 struct page *page = req->wb_page;
Fred Isaman584aa812012-04-20 14:47:51 -0400424
Jeff Laytonbe7e9852012-12-12 12:36:31 -0500425 if (!PageCompound(page) && bytes < hdr->good_bytes)
426 set_page_dirty(page);
Trond Myklebust4bd8b012012-05-01 12:49:58 -0400427 bytes += req->wb_bytes;
428 nfs_list_remove_request(req);
429 nfs_direct_readpage_release(req);
Fred Isaman584aa812012-04-20 14:47:51 -0400430 }
431out_put:
Trond Myklebust607f31e2006-06-28 16:52:45 -0400432 if (put_dreq(dreq))
Trond Myklebustf7b5c342016-06-23 09:29:47 -0400433 nfs_direct_complete(dreq);
Fred Isaman584aa812012-04-20 14:47:51 -0400434 hdr->release(hdr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700435}
436
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400437static void nfs_read_sync_pgio_error(struct list_head *head)
Fred Isamancd841602012-04-20 14:47:44 -0400438{
Fred Isaman584aa812012-04-20 14:47:51 -0400439 struct nfs_page *req;
Fred Isamancd841602012-04-20 14:47:44 -0400440
Fred Isaman584aa812012-04-20 14:47:51 -0400441 while (!list_empty(head)) {
442 req = nfs_list_entry(head->next);
443 nfs_list_remove_request(req);
444 nfs_release_request(req);
445 }
Fred Isamancd841602012-04-20 14:47:44 -0400446}
447
Fred Isaman584aa812012-04-20 14:47:51 -0400448static void nfs_direct_pgio_init(struct nfs_pgio_header *hdr)
449{
450 get_dreq(hdr->dreq);
451}
452
453static const struct nfs_pgio_completion_ops nfs_direct_read_completion_ops = {
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400454 .error_cleanup = nfs_read_sync_pgio_error,
Fred Isaman584aa812012-04-20 14:47:51 -0400455 .init_hdr = nfs_direct_pgio_init,
456 .completion = nfs_direct_read_completion,
457};
458
Chuck Leverd4cc9482006-03-20 13:44:28 -0500459/*
Trond Myklebust607f31e2006-06-28 16:52:45 -0400460 * For each rsize'd chunk of the user's buffer, dispatch an NFS READ
461 * operation. If nfs_readdata_alloc() or get_user_pages() fails,
462 * bail and stop sending more reads. Read length accounting is
463 * handled automatically by nfs_direct_read_result(). Otherwise, if
464 * no requests have been sent, just return an error.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465 */
Al Viro91f79c42014-03-21 04:58:33 -0400466
467static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
468 struct iov_iter *iter,
469 loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700470{
Al Viro91f79c42014-03-21 04:58:33 -0400471 struct nfs_pageio_descriptor desc;
472 struct inode *inode = dreq->inode;
473 ssize_t result = -EINVAL;
474 size_t requested_bytes = 0;
475 size_t rsize = max_t(size_t, NFS_SERVER(inode)->rsize, PAGE_SIZE);
Chuck Lever82b145c2006-06-20 12:57:03 -0400476
Linus Torvalds16b90572014-06-12 10:30:18 -0700477 nfs_pageio_init_read(&desc, dreq->inode, false,
Al Viro91f79c42014-03-21 04:58:33 -0400478 &nfs_direct_read_completion_ops);
479 get_dreq(dreq);
480 desc.pg_dreq = dreq;
Jens Axboefe0f07d2015-04-15 17:05:48 -0600481 inode_dio_begin(inode);
Al Viro91f79c42014-03-21 04:58:33 -0400482
483 while (iov_iter_count(iter)) {
484 struct page **pagevec;
Chuck Lever5dd602f2006-03-20 13:44:29 -0500485 size_t bytes;
Al Viro91f79c42014-03-21 04:58:33 -0400486 size_t pgbase;
487 unsigned npages, i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700488
Al Viro91f79c42014-03-21 04:58:33 -0400489 result = iov_iter_get_pages_alloc(iter, &pagevec,
490 rsize, &pgbase);
491 if (result < 0)
Trond Myklebust607f31e2006-06-28 16:52:45 -0400492 break;
Al Viro91f79c42014-03-21 04:58:33 -0400493
494 bytes = result;
495 iov_iter_advance(iter, bytes);
496 npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
Fred Isaman584aa812012-04-20 14:47:51 -0400497 for (i = 0; i < npages; i++) {
498 struct nfs_page *req;
Trond Myklebustbf5fc402012-05-04 13:47:16 -0400499 unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
Fred Isaman584aa812012-04-20 14:47:51 -0400500 /* XXX do we need to do the eof zeroing found in async_filler? */
Weston Andros Adamson2bfc6e52014-05-15 11:56:45 -0400501 req = nfs_create_request(dreq->ctx, pagevec[i], NULL,
Fred Isaman584aa812012-04-20 14:47:51 -0400502 pgbase, req_len);
503 if (IS_ERR(req)) {
Fred Isaman584aa812012-04-20 14:47:51 -0400504 result = PTR_ERR(req);
505 break;
506 }
507 req->wb_index = pos >> PAGE_SHIFT;
508 req->wb_offset = pos & ~PAGE_MASK;
Al Viro91f79c42014-03-21 04:58:33 -0400509 if (!nfs_pageio_add_request(&desc, req)) {
510 result = desc.pg_error;
Fred Isaman584aa812012-04-20 14:47:51 -0400511 nfs_release_request(req);
Fred Isaman584aa812012-04-20 14:47:51 -0400512 break;
513 }
514 pgbase = 0;
515 bytes -= req_len;
Al Viro91f79c42014-03-21 04:58:33 -0400516 requested_bytes += req_len;
Fred Isaman584aa812012-04-20 14:47:51 -0400517 pos += req_len;
Peng Tao35754bc2012-09-25 14:55:57 +0800518 dreq->bytes_left -= req_len;
Fred Isaman584aa812012-04-20 14:47:51 -0400519 }
Trond Myklebust6d747432012-04-30 13:27:31 -0400520 nfs_direct_release_pages(pagevec, npages);
Al Viro91f79c42014-03-21 04:58:33 -0400521 kvfree(pagevec);
Chuck Lever19f73782007-11-12 12:16:47 -0500522 if (result < 0)
523 break;
Chuck Lever19f73782007-11-12 12:16:47 -0500524 }
525
Fred Isaman584aa812012-04-20 14:47:51 -0400526 nfs_pageio_complete(&desc);
527
Chuck Lever839f7ad2011-01-21 15:54:57 +0000528 /*
529 * If no bytes were started, return the error, and let the
530 * generic layer handle the completion.
531 */
532 if (requested_bytes == 0) {
Jens Axboefe0f07d2015-04-15 17:05:48 -0600533 inode_dio_end(inode);
Chuck Lever839f7ad2011-01-21 15:54:57 +0000534 nfs_direct_req_release(dreq);
535 return result < 0 ? result : -EIO;
536 }
537
Chuck Lever19f73782007-11-12 12:16:47 -0500538 if (put_dreq(dreq))
Trond Myklebustf7b5c342016-06-23 09:29:47 -0400539 nfs_direct_complete(dreq);
Chuck Lever839f7ad2011-01-21 15:54:57 +0000540 return 0;
Chuck Lever19f73782007-11-12 12:16:47 -0500541}
542
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800543/**
544 * nfs_file_direct_read - file direct read operation for NFS files
545 * @iocb: target I/O control block
Al Viro619d30b2014-03-04 21:53:33 -0500546 * @iter: vector of user buffers into which to read data
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800547 *
548 * We use this function for direct reads instead of calling
549 * generic_file_aio_read() in order to avoid gfar's check to see if
550 * the request starts before the end of the file. For that check
551 * to work, we must generate a GETATTR before each direct read, and
552 * even then there is a window between the GETATTR and the subsequent
553 * READ where the file size could change. Our preference is simply
554 * to do all reads the application wants, and the server will take
555 * care of managing the end of file boundary.
556 *
557 * This function also eliminates unnecessarily updating the file's
558 * atime locally, as the NFS server sets the file's atime, and this
559 * client must read the updated atime from the server back into its
560 * cache.
561 */
Christoph Hellwigc8b8e322016-04-07 08:51:58 -0700562ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700563{
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800564 struct file *file = iocb->ki_filp;
565 struct address_space *mapping = file->f_mapping;
566 struct inode *inode = mapping->host;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700567 struct nfs_direct_req *dreq;
Trond Myklebustb3c54de2012-08-13 17:15:50 -0400568 struct nfs_lock_context *l_ctx;
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800569 ssize_t result = -EINVAL;
Al Viroa6cbcd42014-03-04 22:38:00 -0500570 size_t count = iov_iter_count(iter);
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800571 nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count);
572
573 dfprintk(FILE, "NFS: direct read(%pD2, %zd@%Ld)\n",
Christoph Hellwigc8b8e322016-04-07 08:51:58 -0700574 file, count, (long long) iocb->ki_pos);
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800575
576 result = 0;
577 if (!count)
578 goto out;
579
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800580 task_io_account_read(count);
581
582 result = -ENOMEM;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400583 dreq = nfs_direct_req_alloc();
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400584 if (dreq == NULL)
Trond Myklebusta5864c92016-06-03 17:07:19 -0400585 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700586
Chuck Lever91d5b472006-03-20 13:44:14 -0500587 dreq->inode = inode;
Weston Andros Adamsoned3743a2016-04-01 11:42:29 -0400588 dreq->bytes_left = dreq->max_count = count;
Christoph Hellwigc8b8e322016-04-07 08:51:58 -0700589 dreq->io_start = iocb->ki_pos;
Trond Myklebustcd3758e2007-08-10 17:44:32 -0400590 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
Trond Myklebustb3c54de2012-08-13 17:15:50 -0400591 l_ctx = nfs_get_lock_context(dreq->ctx);
592 if (IS_ERR(l_ctx)) {
593 result = PTR_ERR(l_ctx);
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400594 goto out_release;
Trond Myklebustb3c54de2012-08-13 17:15:50 -0400595 }
596 dreq->l_ctx = l_ctx;
Chuck Lever487b8372006-03-20 13:44:30 -0500597 if (!is_sync_kiocb(iocb))
598 dreq->iocb = iocb;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700599
Trond Myklebusta5864c92016-06-03 17:07:19 -0400600 nfs_start_io_direct(inode);
601
Al Viro619d30b2014-03-04 21:53:33 -0500602 NFS_I(inode)->read_io += count;
Christoph Hellwigc8b8e322016-04-07 08:51:58 -0700603 result = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos);
Christoph Hellwigd0b98752013-11-14 08:50:33 -0800604
Trond Myklebusta5864c92016-06-03 17:07:19 -0400605 nfs_end_io_direct(inode);
Christoph Hellwigd0b98752013-11-14 08:50:33 -0800606
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800607 if (!result) {
Trond Myklebust607f31e2006-06-28 16:52:45 -0400608 result = nfs_direct_wait(dreq);
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800609 if (result > 0)
Christoph Hellwigc8b8e322016-04-07 08:51:58 -0700610 iocb->ki_pos += result;
Christoph Hellwig14a3ec792013-11-14 08:50:31 -0800611 }
Christoph Hellwigd0b98752013-11-14 08:50:33 -0800612
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400613out_release:
Trond Myklebustb4946ff2007-05-30 12:58:00 -0400614 nfs_direct_req_release(dreq);
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400615out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700616 return result;
617}
618
Tom Haynes085d1e32014-12-11 13:04:55 -0500619static void
620nfs_direct_write_scan_commit_list(struct inode *inode,
621 struct list_head *list,
622 struct nfs_commit_info *cinfo)
623{
Dave Wysochanskife238e62016-04-01 13:45:09 -0400624 spin_lock(&cinfo->inode->i_lock);
Tom Haynes085d1e32014-12-11 13:04:55 -0500625#ifdef CONFIG_NFS_V4_1
626 if (cinfo->ds != NULL && cinfo->ds->nwritten != 0)
627 NFS_SERVER(inode)->pnfs_curr_ld->recover_commit_reqs(list, cinfo);
628#endif
629 nfs_scan_commit_list(&cinfo->mds->list, list, cinfo, 0);
Dave Wysochanskife238e62016-04-01 13:45:09 -0400630 spin_unlock(&cinfo->inode->i_lock);
Tom Haynes085d1e32014-12-11 13:04:55 -0500631}
632
Trond Myklebustfad61492006-03-20 13:44:36 -0500633static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634{
Fred Isaman1763da12012-04-20 14:47:57 -0400635 struct nfs_pageio_descriptor desc;
636 struct nfs_page *req, *tmp;
637 LIST_HEAD(reqs);
638 struct nfs_commit_info cinfo;
639 LIST_HEAD(failed);
Weston Andros Adamson0a00b772014-09-19 12:48:33 -0400640 int i;
Fred Isaman1763da12012-04-20 14:47:57 -0400641
642 nfs_init_cinfo_from_dreq(&cinfo, dreq);
Tom Haynes085d1e32014-12-11 13:04:55 -0500643 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700644
Trond Myklebustfad61492006-03-20 13:44:36 -0500645 dreq->count = 0;
Trond Myklebusta5314a72016-06-01 21:42:32 -0400646 dreq->verf.committed = NFS_INVALID_STABLE_HOW;
647 nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo);
Weston Andros Adamson0a00b772014-09-19 12:48:33 -0400648 for (i = 0; i < dreq->mirror_count; i++)
649 dreq->mirrors[i].count = 0;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400650 get_dreq(dreq);
Trond Myklebustfad61492006-03-20 13:44:36 -0500651
Christoph Hellwiga20c93e2014-04-16 15:07:21 +0200652 nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,
Fred Isaman1763da12012-04-20 14:47:57 -0400653 &nfs_direct_write_completion_ops);
654 desc.pg_dreq = dreq;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400655
Weston Andros Adamson0a00b772014-09-19 12:48:33 -0400656 req = nfs_list_entry(reqs.next);
657 nfs_direct_setup_mirroring(dreq, &desc, req);
Peng Taod600ad12015-12-04 02:57:48 +0800658 if (desc.pg_error < 0) {
659 list_splice_init(&reqs, &failed);
660 goto out_failed;
661 }
Weston Andros Adamson0a00b772014-09-19 12:48:33 -0400662
Fred Isaman1763da12012-04-20 14:47:57 -0400663 list_for_each_entry_safe(req, tmp, &reqs, wb_list) {
664 if (!nfs_pageio_add_request(&desc, req)) {
Trond Myklebust4035c242012-07-08 10:24:10 -0400665 nfs_list_remove_request(req);
Fred Isaman1763da12012-04-20 14:47:57 -0400666 nfs_list_add_request(req, &failed);
Dave Wysochanskife238e62016-04-01 13:45:09 -0400667 spin_lock(&cinfo.inode->i_lock);
Fred Isaman1763da12012-04-20 14:47:57 -0400668 dreq->flags = 0;
Peng Taod600ad12015-12-04 02:57:48 +0800669 if (desc.pg_error < 0)
670 dreq->error = desc.pg_error;
671 else
672 dreq->error = -EIO;
Dave Wysochanskife238e62016-04-01 13:45:09 -0400673 spin_unlock(&cinfo.inode->i_lock);
Fred Isaman1763da12012-04-20 14:47:57 -0400674 }
Trond Myklebust5a695da2012-06-19 13:39:14 -0400675 nfs_release_request(req);
Fred Isaman1763da12012-04-20 14:47:57 -0400676 }
677 nfs_pageio_complete(&desc);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400678
Peng Taod600ad12015-12-04 02:57:48 +0800679out_failed:
Trond Myklebust4035c242012-07-08 10:24:10 -0400680 while (!list_empty(&failed)) {
681 req = nfs_list_entry(failed.next);
682 nfs_list_remove_request(req);
Trond Myklebust1d1afcb2012-05-09 14:04:55 -0400683 nfs_unlock_and_release_request(req);
Trond Myklebust4035c242012-07-08 10:24:10 -0400684 }
Trond Myklebust607f31e2006-06-28 16:52:45 -0400685
686 if (put_dreq(dreq))
Fred Isaman1763da12012-04-20 14:47:57 -0400687 nfs_direct_write_complete(dreq, dreq->inode);
Trond Myklebustfad61492006-03-20 13:44:36 -0500688}
689
Fred Isaman1763da12012-04-20 14:47:57 -0400690static void nfs_direct_commit_complete(struct nfs_commit_data *data)
Trond Myklebustfad61492006-03-20 13:44:36 -0500691{
Fred Isaman0b7c0152012-04-20 14:47:39 -0400692 struct nfs_direct_req *dreq = data->dreq;
Fred Isaman1763da12012-04-20 14:47:57 -0400693 struct nfs_commit_info cinfo;
694 struct nfs_page *req;
Trond Myklebustc9d8f892008-04-15 16:56:39 -0400695 int status = data->task.tk_status;
696
Fred Isaman1763da12012-04-20 14:47:57 -0400697 nfs_init_cinfo_from_dreq(&cinfo, dreq);
Trond Myklebustc9d8f892008-04-15 16:56:39 -0400698 if (status < 0) {
Trond Myklebust60fa3f72007-04-14 19:11:52 -0400699 dprintk("NFS: %5u commit failed with error %d.\n",
Fred Isaman1763da12012-04-20 14:47:57 -0400700 data->task.tk_pid, status);
Trond Myklebustfad61492006-03-20 13:44:36 -0500701 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400702 } else if (nfs_direct_cmp_commit_data_verf(dreq, data)) {
Trond Myklebustc9d8f892008-04-15 16:56:39 -0400703 dprintk("NFS: %5u commit verify failed\n", data->task.tk_pid);
Trond Myklebustfad61492006-03-20 13:44:36 -0500704 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
705 }
706
Trond Myklebustc9d8f892008-04-15 16:56:39 -0400707 dprintk("NFS: %5u commit returned %d\n", data->task.tk_pid, status);
Fred Isaman1763da12012-04-20 14:47:57 -0400708 while (!list_empty(&data->pages)) {
709 req = nfs_list_entry(data->pages.next);
710 nfs_list_remove_request(req);
711 if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) {
712 /* Note the rewrite will go through mds */
Weston Andros Adamsonb57ff132014-09-05 18:20:21 -0400713 nfs_mark_request_commit(req, NULL, &cinfo, 0);
Fred Isaman906369e2012-06-08 16:48:33 -0400714 } else
715 nfs_release_request(req);
Trond Myklebust1d1afcb2012-05-09 14:04:55 -0400716 nfs_unlock_and_release_request(req);
Fred Isaman1763da12012-04-20 14:47:57 -0400717 }
718
719 if (atomic_dec_and_test(&cinfo.mds->rpcs_out))
720 nfs_direct_write_complete(dreq, data->inode);
Trond Myklebustfad61492006-03-20 13:44:36 -0500721}
722
Trond Myklebustb20135d2015-12-31 09:28:06 -0500723static void nfs_direct_resched_write(struct nfs_commit_info *cinfo,
724 struct nfs_page *req)
Fred Isaman1763da12012-04-20 14:47:57 -0400725{
Trond Myklebustb20135d2015-12-31 09:28:06 -0500726 struct nfs_direct_req *dreq = cinfo->dreq;
727
728 spin_lock(&dreq->lock);
729 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
730 spin_unlock(&dreq->lock);
731 nfs_mark_request_commit(req, NULL, cinfo, 0);
Fred Isaman1763da12012-04-20 14:47:57 -0400732}
733
734static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops = {
735 .completion = nfs_direct_commit_complete,
Trond Myklebustb20135d2015-12-31 09:28:06 -0500736 .resched_write = nfs_direct_resched_write,
Trond Myklebustfad61492006-03-20 13:44:36 -0500737};
738
739static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
740{
Fred Isaman1763da12012-04-20 14:47:57 -0400741 int res;
742 struct nfs_commit_info cinfo;
743 LIST_HEAD(mds_list);
Trond Myklebustfad61492006-03-20 13:44:36 -0500744
Fred Isaman1763da12012-04-20 14:47:57 -0400745 nfs_init_cinfo_from_dreq(&cinfo, dreq);
746 nfs_scan_commit(dreq->inode, &mds_list, &cinfo);
747 res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);
748 if (res < 0) /* res == -ENOMEM */
749 nfs_direct_write_reschedule(dreq);
Trond Myklebustfad61492006-03-20 13:44:36 -0500750}
751
Fred Isaman1763da12012-04-20 14:47:57 -0400752static void nfs_direct_write_schedule_work(struct work_struct *work)
Trond Myklebustfad61492006-03-20 13:44:36 -0500753{
Fred Isaman1763da12012-04-20 14:47:57 -0400754 struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);
Trond Myklebustfad61492006-03-20 13:44:36 -0500755 int flags = dreq->flags;
756
757 dreq->flags = 0;
758 switch (flags) {
759 case NFS_ODIRECT_DO_COMMIT:
760 nfs_direct_commit_schedule(dreq);
761 break;
762 case NFS_ODIRECT_RESCHED_WRITES:
763 nfs_direct_write_reschedule(dreq);
764 break;
765 default:
Trond Myklebustf7b5c342016-06-23 09:29:47 -0400766 nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping);
767 nfs_direct_complete(dreq);
Trond Myklebustfad61492006-03-20 13:44:36 -0500768 }
769}
770
Fred Isaman1763da12012-04-20 14:47:57 -0400771static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
Trond Myklebustfad61492006-03-20 13:44:36 -0500772{
Fred Isaman1763da12012-04-20 14:47:57 -0400773 schedule_work(&dreq->work); /* Calls nfs_direct_write_schedule_work */
Trond Myklebustfad61492006-03-20 13:44:36 -0500774}
Fred Isaman1763da12012-04-20 14:47:57 -0400775
Fred Isaman1763da12012-04-20 14:47:57 -0400776static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
777{
778 struct nfs_direct_req *dreq = hdr->dreq;
779 struct nfs_commit_info cinfo;
Weston Andros Adamsonc65e6252014-06-09 11:48:36 -0400780 bool request_commit = false;
Fred Isaman1763da12012-04-20 14:47:57 -0400781 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
782
783 if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
784 goto out_put;
785
786 nfs_init_cinfo_from_dreq(&cinfo, dreq);
787
788 spin_lock(&dreq->lock);
789
J. Bruce Fieldsca2c3162018-01-16 10:08:00 -0500790 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags))
Fred Isaman1763da12012-04-20 14:47:57 -0400791 dreq->error = hdr->error;
Weston Andros Adamsonc65e6252014-06-09 11:48:36 -0400792 if (dreq->error == 0) {
Weston Andros Adamson0a00b772014-09-19 12:48:33 -0400793 nfs_direct_good_bytes(dreq, hdr);
Weston Andros Adamsonc65e6252014-06-09 11:48:36 -0400794 if (nfs_write_need_commit(hdr)) {
Fred Isaman1763da12012-04-20 14:47:57 -0400795 if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES)
Weston Andros Adamsonc65e6252014-06-09 11:48:36 -0400796 request_commit = true;
Fred Isaman1763da12012-04-20 14:47:57 -0400797 else if (dreq->flags == 0) {
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400798 nfs_direct_set_hdr_verf(dreq, hdr);
Weston Andros Adamsonc65e6252014-06-09 11:48:36 -0400799 request_commit = true;
Fred Isaman1763da12012-04-20 14:47:57 -0400800 dreq->flags = NFS_ODIRECT_DO_COMMIT;
801 } else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) {
Weston Andros Adamsonc65e6252014-06-09 11:48:36 -0400802 request_commit = true;
803 if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr))
Weston Andros Adamson5002c582014-05-15 11:56:54 -0400804 dreq->flags =
805 NFS_ODIRECT_RESCHED_WRITES;
Fred Isaman1763da12012-04-20 14:47:57 -0400806 }
807 }
808 }
809 spin_unlock(&dreq->lock);
810
811 while (!list_empty(&hdr->pages)) {
Weston Andros Adamson2bfc6e52014-05-15 11:56:45 -0400812
Fred Isaman1763da12012-04-20 14:47:57 -0400813 req = nfs_list_entry(hdr->pages.next);
814 nfs_list_remove_request(req);
Weston Andros Adamsonc65e6252014-06-09 11:48:36 -0400815 if (request_commit) {
Trond Myklebust04277082012-05-09 13:54:53 -0400816 kref_get(&req->wb_kref);
Weston Andros Adamsonb57ff132014-09-05 18:20:21 -0400817 nfs_mark_request_commit(req, hdr->lseg, &cinfo,
818 hdr->ds_commit_idx);
Fred Isaman1763da12012-04-20 14:47:57 -0400819 }
Trond Myklebust1d1afcb2012-05-09 14:04:55 -0400820 nfs_unlock_and_release_request(req);
Fred Isaman1763da12012-04-20 14:47:57 -0400821 }
822
823out_put:
824 if (put_dreq(dreq))
825 nfs_direct_write_complete(dreq, hdr->inode);
826 hdr->release(hdr);
827}
828
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400829static void nfs_write_sync_pgio_error(struct list_head *head)
830{
831 struct nfs_page *req;
832
833 while (!list_empty(head)) {
834 req = nfs_list_entry(head->next);
835 nfs_list_remove_request(req);
Trond Myklebust1d1afcb2012-05-09 14:04:55 -0400836 nfs_unlock_and_release_request(req);
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400837 }
838}
839
Trond Myklebustdc602dd2015-12-31 11:44:06 -0500840static void nfs_direct_write_reschedule_io(struct nfs_pgio_header *hdr)
841{
842 struct nfs_direct_req *dreq = hdr->dreq;
843
844 spin_lock(&dreq->lock);
845 if (dreq->error == 0) {
846 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
847 /* fake unstable write to let common nfs resend pages */
848 hdr->verf.committed = NFS_UNSTABLE;
849 hdr->good_bytes = hdr->args.count;
850 }
851 spin_unlock(&dreq->lock);
852}
853
Fred Isaman1763da12012-04-20 14:47:57 -0400854static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops = {
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400855 .error_cleanup = nfs_write_sync_pgio_error,
Fred Isaman1763da12012-04-20 14:47:57 -0400856 .init_hdr = nfs_direct_pgio_init,
857 .completion = nfs_direct_write_completion,
Trond Myklebustdc602dd2015-12-31 11:44:06 -0500858 .reschedule_io = nfs_direct_write_reschedule_io,
Fred Isaman1763da12012-04-20 14:47:57 -0400859};
860
Al Viro91f79c42014-03-21 04:58:33 -0400861
862/*
863 * NB: Return the value of the first error return code. Subsequent
864 * errors after the first one are ignored.
865 */
866/*
867 * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE
868 * operation. If nfs_writedata_alloc() or get_user_pages() fails,
869 * bail and stop sending more writes. Write length accounting is
870 * handled automatically by nfs_direct_write_result(). Otherwise, if
871 * no requests have been sent, just return an error.
872 */
Chuck Lever19f73782007-11-12 12:16:47 -0500873static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
Al Viro619d30b2014-03-04 21:53:33 -0500874 struct iov_iter *iter,
Al Viro91f79c42014-03-21 04:58:33 -0400875 loff_t pos)
Chuck Lever19f73782007-11-12 12:16:47 -0500876{
Fred Isaman1763da12012-04-20 14:47:57 -0400877 struct nfs_pageio_descriptor desc;
Trond Myklebust1d59d612012-05-31 12:22:33 -0400878 struct inode *inode = dreq->inode;
Chuck Lever19f73782007-11-12 12:16:47 -0500879 ssize_t result = 0;
880 size_t requested_bytes = 0;
Al Viro91f79c42014-03-21 04:58:33 -0400881 size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE);
Chuck Lever19f73782007-11-12 12:16:47 -0500882
Christoph Hellwiga20c93e2014-04-16 15:07:21 +0200883 nfs_pageio_init_write(&desc, inode, FLUSH_COND_STABLE, false,
Fred Isaman1763da12012-04-20 14:47:57 -0400884 &nfs_direct_write_completion_ops);
885 desc.pg_dreq = dreq;
Chuck Lever19f73782007-11-12 12:16:47 -0500886 get_dreq(dreq);
Jens Axboefe0f07d2015-04-15 17:05:48 -0600887 inode_dio_begin(inode);
Chuck Lever19f73782007-11-12 12:16:47 -0500888
Al Viro91f79c42014-03-21 04:58:33 -0400889 NFS_I(inode)->write_io += iov_iter_count(iter);
890 while (iov_iter_count(iter)) {
891 struct page **pagevec;
892 size_t bytes;
893 size_t pgbase;
894 unsigned npages, i;
895
896 result = iov_iter_get_pages_alloc(iter, &pagevec,
897 wsize, &pgbase);
Chuck Lever19f73782007-11-12 12:16:47 -0500898 if (result < 0)
899 break;
Al Viro91f79c42014-03-21 04:58:33 -0400900
901 bytes = result;
902 iov_iter_advance(iter, bytes);
903 npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
904 for (i = 0; i < npages; i++) {
905 struct nfs_page *req;
906 unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
907
Linus Torvalds16b90572014-06-12 10:30:18 -0700908 req = nfs_create_request(dreq->ctx, pagevec[i], NULL,
Al Viro91f79c42014-03-21 04:58:33 -0400909 pgbase, req_len);
910 if (IS_ERR(req)) {
911 result = PTR_ERR(req);
912 break;
913 }
Weston Andros Adamson0a00b772014-09-19 12:48:33 -0400914
915 nfs_direct_setup_mirroring(dreq, &desc, req);
Peng Taod600ad12015-12-04 02:57:48 +0800916 if (desc.pg_error < 0) {
917 nfs_free_request(req);
918 result = desc.pg_error;
919 break;
920 }
Weston Andros Adamson0a00b772014-09-19 12:48:33 -0400921
Al Viro91f79c42014-03-21 04:58:33 -0400922 nfs_lock_request(req);
923 req->wb_index = pos >> PAGE_SHIFT;
924 req->wb_offset = pos & ~PAGE_MASK;
925 if (!nfs_pageio_add_request(&desc, req)) {
926 result = desc.pg_error;
927 nfs_unlock_and_release_request(req);
928 break;
929 }
930 pgbase = 0;
931 bytes -= req_len;
932 requested_bytes += req_len;
933 pos += req_len;
934 dreq->bytes_left -= req_len;
935 }
936 nfs_direct_release_pages(pagevec, npages);
937 kvfree(pagevec);
938 if (result < 0)
Chuck Lever19f73782007-11-12 12:16:47 -0500939 break;
Chuck Lever19f73782007-11-12 12:16:47 -0500940 }
Fred Isaman1763da12012-04-20 14:47:57 -0400941 nfs_pageio_complete(&desc);
Chuck Lever19f73782007-11-12 12:16:47 -0500942
Chuck Lever839f7ad2011-01-21 15:54:57 +0000943 /*
944 * If no bytes were started, return the error, and let the
945 * generic layer handle the completion.
946 */
947 if (requested_bytes == 0) {
Jens Axboefe0f07d2015-04-15 17:05:48 -0600948 inode_dio_end(inode);
Chuck Lever839f7ad2011-01-21 15:54:57 +0000949 nfs_direct_req_release(dreq);
950 return result < 0 ? result : -EIO;
951 }
952
Chuck Lever19f73782007-11-12 12:16:47 -0500953 if (put_dreq(dreq))
954 nfs_direct_write_complete(dreq, dreq->inode);
Chuck Lever839f7ad2011-01-21 15:54:57 +0000955 return 0;
Chuck Lever19f73782007-11-12 12:16:47 -0500956}
957
Linus Torvalds1da177e2005-04-16 15:20:36 -0700958/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959 * nfs_file_direct_write - file direct write operation for NFS files
960 * @iocb: target I/O control block
Al Viro619d30b2014-03-04 21:53:33 -0500961 * @iter: vector of user buffers from which to write data
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962 *
963 * We use this function for direct writes instead of calling
964 * generic_file_aio_write() in order to avoid taking the inode
965 * semaphore and updating the i_size. The NFS server will set
966 * the new i_size and this client must read the updated size
967 * back into its cache. We let the server do generic write
968 * parameter checking and report problems.
969 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700970 * We eliminate local atime updates, see direct read above.
971 *
972 * We avoid unnecessary page cache invalidations for normal cached
973 * readers of this file.
974 *
975 * Note that O_APPEND is not supported for NFS direct writes, as there
976 * is no atomic O_APPEND write facility in the NFS protocol.
977 */
Al Viro65a4a1c2015-04-09 14:11:08 -0400978ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979{
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -0800980 ssize_t result = -EINVAL;
Trond Myklebust89698b22016-06-23 10:35:48 -0400981 size_t count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700982 struct file *file = iocb->ki_filp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700983 struct address_space *mapping = file->f_mapping;
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -0800984 struct inode *inode = mapping->host;
985 struct nfs_direct_req *dreq;
986 struct nfs_lock_context *l_ctx;
Al Viro65a4a1c2015-04-09 14:11:08 -0400987 loff_t pos, end;
Chuck Leverc216fd72007-11-12 12:16:52 -0500988
Al Viro6de14722013-09-16 10:53:17 -0400989 dfprintk(FILE, "NFS: direct write(%pD2, %zd@%Ld)\n",
Al Viro3309dd02015-04-09 12:55:47 -0400990 file, iov_iter_count(iter), (long long) iocb->ki_pos);
Badari Pulavarty027445c2006-09-30 23:28:46 -0700991
Trond Myklebust89698b22016-06-23 10:35:48 -0400992 result = generic_write_checks(iocb, iter);
993 if (result <= 0)
994 return result;
995 count = result;
996 nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count);
Al Viro3309dd02015-04-09 12:55:47 -0400997
998 pos = iocb->ki_pos;
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +0300999 end = (pos + iov_iter_count(iter) - 1) >> PAGE_SHIFT;
Chuck Leverce1a8e62005-11-30 18:08:17 -05001000
Trond Myklebust89698b22016-06-23 10:35:48 -04001001 task_io_account_write(count);
Konstantin Khlebnikov7ec10f22011-02-22 00:28:34 +03001002
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -08001003 result = -ENOMEM;
1004 dreq = nfs_direct_req_alloc();
1005 if (!dreq)
Trond Myklebusta5864c92016-06-03 17:07:19 -04001006 goto out;
Chuck Lever9eafa8c2006-03-20 13:44:33 -05001007
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -08001008 dreq->inode = inode;
Trond Myklebust89698b22016-06-23 10:35:48 -04001009 dreq->bytes_left = dreq->max_count = count;
Peng Tao5fadeb42015-01-19 12:41:16 +08001010 dreq->io_start = pos;
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -08001011 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
1012 l_ctx = nfs_get_lock_context(dreq->ctx);
1013 if (IS_ERR(l_ctx)) {
1014 result = PTR_ERR(l_ctx);
1015 goto out_release;
Fred Isaman1763da12012-04-20 14:47:57 -04001016 }
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -08001017 dreq->l_ctx = l_ctx;
1018 if (!is_sync_kiocb(iocb))
1019 dreq->iocb = iocb;
1020
Trond Myklebusta5864c92016-06-03 17:07:19 -04001021 nfs_start_io_direct(inode);
1022
Al Viro91f79c42014-03-21 04:58:33 -04001023 result = nfs_direct_write_schedule_iovec(dreq, iter, pos);
Christoph Hellwiga9ab5e82013-11-14 08:50:34 -08001024
1025 if (mapping->nrpages) {
1026 invalidate_inode_pages2_range(mapping,
Kirill A. Shutemov09cbfea2016-04-01 15:29:47 +03001027 pos >> PAGE_SHIFT, end);
Christoph Hellwiga9ab5e82013-11-14 08:50:34 -08001028 }
1029
Trond Myklebusta5864c92016-06-03 17:07:19 -04001030 nfs_end_io_direct(inode);
Christoph Hellwiga9ab5e82013-11-14 08:50:34 -08001031
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -08001032 if (!result) {
1033 result = nfs_direct_wait(dreq);
1034 if (result > 0) {
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -08001035 iocb->ki_pos = pos + result;
Christoph Hellwige2592212016-04-07 08:52:01 -07001036 /* XXX: should check the generic_write_sync retval */
1037 generic_write_sync(iocb, result);
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -08001038 }
1039 }
1040out_release:
1041 nfs_direct_req_release(dreq);
Trond Myklebusta5864c92016-06-03 17:07:19 -04001042out:
Christoph Hellwig22cd1bf2013-11-14 08:50:32 -08001043 return result;
Linus Torvalds1da177e2005-04-16 15:20:36 -07001044}
1045
Chuck Lever88467052006-03-20 13:44:34 -05001046/**
1047 * nfs_init_directcache - create a slab cache for nfs_direct_req structures
1048 *
1049 */
David Howellsf7b422b2006-06-09 09:34:33 -04001050int __init nfs_init_directcache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001051{
1052 nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
1053 sizeof(struct nfs_direct_req),
Paul Jacksonfffb60f2006-03-24 03:16:06 -08001054 0, (SLAB_RECLAIM_ACCOUNT|
1055 SLAB_MEM_SPREAD),
Paul Mundt20c2df82007-07-20 10:11:58 +09001056 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001057 if (nfs_direct_cachep == NULL)
1058 return -ENOMEM;
1059
1060 return 0;
1061}
1062
Chuck Lever88467052006-03-20 13:44:34 -05001063/**
David Howellsf7b422b2006-06-09 09:34:33 -04001064 * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures
Chuck Lever88467052006-03-20 13:44:34 -05001065 *
1066 */
David Brownell266bee82006-06-27 12:59:15 -07001067void nfs_destroy_directcache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001068{
Alexey Dobriyan1a1d92c2006-09-27 01:49:40 -07001069 kmem_cache_destroy(nfs_direct_cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001070}