blob: 91ff089d34126d8ae5d8d4a96bca232fbd921958 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/nfs/direct.c
3 *
4 * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
5 *
6 * High-performance uncached I/O for the Linux NFS client
7 *
8 * There are important applications whose performance or correctness
9 * depends on uncached access to file data. Database clusters
Chuck Lever88467052006-03-20 13:44:34 -050010 * (multiple copies of the same instance running on separate hosts)
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * implement their own cache coherency protocol that subsumes file
Chuck Lever88467052006-03-20 13:44:34 -050012 * system cache protocols. Applications that process datasets
13 * considerably larger than the client's memory do not always benefit
14 * from a local cache. A streaming video server, for instance, has no
Linus Torvalds1da177e2005-04-16 15:20:36 -070015 * need to cache the contents of a file.
16 *
17 * When an application requests uncached I/O, all read and write requests
18 * are made directly to the server; data stored or fetched via these
19 * requests is not cached in the Linux page cache. The client does not
20 * correct unaligned requests from applications. All requested bytes are
21 * held on permanent storage before a direct write system call returns to
22 * an application.
23 *
24 * Solaris implements an uncached I/O facility called directio() that
25 * is used for backups and sequential I/O to very large files. Solaris
26 * also supports uncaching whole NFS partitions with "-o forcedirectio,"
27 * an undocumented mount option.
28 *
29 * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
30 * help from Andrew Morton.
31 *
32 * 18 Dec 2001 Initial implementation for 2.4 --cel
33 * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy
34 * 08 Jun 2003 Port to 2.5 APIs --cel
35 * 31 Mar 2004 Handle direct I/O without VFS support --cel
36 * 15 Sep 2004 Parallel async reads --cel
Chuck Lever88467052006-03-20 13:44:34 -050037 * 04 May 2005 support O_DIRECT with aio --cel
Linus Torvalds1da177e2005-04-16 15:20:36 -070038 *
39 */
40
Linus Torvalds1da177e2005-04-16 15:20:36 -070041#include <linux/errno.h>
42#include <linux/sched.h>
43#include <linux/kernel.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070044#include <linux/file.h>
45#include <linux/pagemap.h>
46#include <linux/kref.h>
Tejun Heo5a0e3ad2010-03-24 17:04:11 +090047#include <linux/slab.h>
Konstantin Khlebnikov7ec10f22011-02-22 00:28:34 +030048#include <linux/task_io_accounting_ops.h>
Peng Tao62965562012-09-25 14:55:57 +080049#include <linux/module.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070050
51#include <linux/nfs_fs.h>
52#include <linux/nfs_page.h>
53#include <linux/sunrpc/clnt.h>
54
Linus Torvalds1da177e2005-04-16 15:20:36 -070055#include <asm/uaccess.h>
Arun Sharma600634972011-07-26 16:09:06 -070056#include <linux/atomic.h>
Linus Torvalds1da177e2005-04-16 15:20:36 -070057
Trond Myklebust8d5658c2007-04-10 09:26:35 -040058#include "internal.h"
Chuck Lever91d5b472006-03-20 13:44:14 -050059#include "iostat.h"
Fred Isaman1763da12012-04-20 14:47:57 -040060#include "pnfs.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070061
Linus Torvalds1da177e2005-04-16 15:20:36 -070062#define NFSDBG_FACILITY NFSDBG_VFS
Linus Torvalds1da177e2005-04-16 15:20:36 -070063
Christoph Lametere18b8902006-12-06 20:33:20 -080064static struct kmem_cache *nfs_direct_cachep;
Linus Torvalds1da177e2005-04-16 15:20:36 -070065
66/*
67 * This represents a set of asynchronous requests that we're waiting on
68 */
69struct nfs_direct_req {
70 struct kref kref; /* release manager */
Chuck Lever15ce4a02006-03-20 13:44:34 -050071
72 /* I/O parameters */
Trond Myklebusta8881f52006-03-20 13:44:36 -050073 struct nfs_open_context *ctx; /* file open context info */
Trond Myklebustf11ac8d2010-06-25 16:35:53 -040074 struct nfs_lock_context *l_ctx; /* Lock context info */
Chuck Lever99514f82006-03-20 13:44:30 -050075 struct kiocb * iocb; /* controlling i/o request */
Chuck Lever88467052006-03-20 13:44:34 -050076 struct inode * inode; /* target file of i/o */
Chuck Lever15ce4a02006-03-20 13:44:34 -050077
78 /* completion state */
Trond Myklebust607f31e2006-06-28 16:52:45 -040079 atomic_t io_count; /* i/os we're waiting for */
Chuck Lever15ce4a02006-03-20 13:44:34 -050080 spinlock_t lock; /* protect completion state */
Chuck Lever15ce4a02006-03-20 13:44:34 -050081 ssize_t count, /* bytes actually processed */
Peng Tao35754bc2012-09-25 14:55:57 +080082 bytes_left, /* bytes left to be sent */
Linus Torvalds1da177e2005-04-16 15:20:36 -070083 error; /* any reported error */
Trond Myklebustd72b7a62006-03-20 13:44:43 -050084 struct completion completion; /* wait for i/o completion */
Trond Myklebustfad61492006-03-20 13:44:36 -050085
86 /* commit state */
Fred Isaman1763da12012-04-20 14:47:57 -040087 struct nfs_mds_commit_info mds_cinfo; /* Storage for cinfo */
88 struct pnfs_ds_commit_info ds_cinfo; /* Storage for cinfo */
89 struct work_struct work;
Trond Myklebustfad61492006-03-20 13:44:36 -050090 int flags;
91#define NFS_ODIRECT_DO_COMMIT (1) /* an unstable reply was received */
92#define NFS_ODIRECT_RESCHED_WRITES (2) /* write verification failed */
93 struct nfs_writeverf verf; /* unstable write verifier */
Linus Torvalds1da177e2005-04-16 15:20:36 -070094};
95
Fred Isaman1763da12012-04-20 14:47:57 -040096static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops;
97static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops;
Trond Myklebustfad61492006-03-20 13:44:36 -050098static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode);
Fred Isaman1763da12012-04-20 14:47:57 -040099static void nfs_direct_write_schedule_work(struct work_struct *work);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400100
101static inline void get_dreq(struct nfs_direct_req *dreq)
102{
103 atomic_inc(&dreq->io_count);
104}
105
106static inline int put_dreq(struct nfs_direct_req *dreq)
107{
108 return atomic_dec_and_test(&dreq->io_count);
109}
110
Linus Torvalds1da177e2005-04-16 15:20:36 -0700111/**
Chuck Leverb8a32e22006-03-20 13:44:28 -0500112 * nfs_direct_IO - NFS address space operation for direct I/O
113 * @rw: direction (read or write)
114 * @iocb: target I/O control block
115 * @iov: array of vectors that define I/O buffer
116 * @pos: offset in file to begin the operation
117 * @nr_segs: size of iovec array
118 *
119 * The presence of this routine in the address space ops vector means
Mel Gormana564b8f2012-07-31 16:45:12 -0700120 * the NFS client supports direct I/O. However, for most direct IO, we
121 * shunt off direct read and write requests before the VFS gets them,
122 * so this method is only ever called for swap.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700123 */
Chuck Leverb8a32e22006-03-20 13:44:28 -0500124ssize_t nfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t pos, unsigned long nr_segs)
125{
Mel Gormana564b8f2012-07-31 16:45:12 -0700126#ifndef CONFIG_NFS_SWAP
Chuck Leverb8a32e22006-03-20 13:44:28 -0500127 dprintk("NFS: nfs_direct_IO (%s) off/no(%Ld/%lu) EINVAL\n",
Josef "Jeff" Sipek01cce932006-12-08 02:36:40 -0800128 iocb->ki_filp->f_path.dentry->d_name.name,
Trond Myklebuste99170f2006-04-18 13:21:42 -0400129 (long long) pos, nr_segs);
Chuck Leverb8a32e22006-03-20 13:44:28 -0500130
131 return -EINVAL;
Mel Gormana564b8f2012-07-31 16:45:12 -0700132#else
Mel Gormana564b8f2012-07-31 16:45:12 -0700133 VM_BUG_ON(iocb->ki_nbytes != PAGE_SIZE);
134
135 if (rw == READ || rw == KERNEL_READ)
136 return nfs_file_direct_read(iocb, iov, nr_segs, pos,
137 rw == READ ? true : false);
138 return nfs_file_direct_write(iocb, iov, nr_segs, pos,
139 rw == WRITE ? true : false);
140#endif /* CONFIG_NFS_SWAP */
Chuck Leverb8a32e22006-03-20 13:44:28 -0500141}
142
Chuck Lever749e1462007-05-19 17:22:46 -0400143static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
Chuck Lever9c93ab72006-06-20 12:56:31 -0400144{
Chuck Lever749e1462007-05-19 17:22:46 -0400145 unsigned int i;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400146 for (i = 0; i < npages; i++)
147 page_cache_release(pages[i]);
Trond Myklebust6b45d852006-03-20 13:44:43 -0500148}
149
Fred Isaman1763da12012-04-20 14:47:57 -0400150void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo,
151 struct nfs_direct_req *dreq)
152{
153 cinfo->lock = &dreq->lock;
154 cinfo->mds = &dreq->mds_cinfo;
155 cinfo->ds = &dreq->ds_cinfo;
156 cinfo->dreq = dreq;
157 cinfo->completion_ops = &nfs_direct_commit_completion_ops;
158}
159
Chuck Lever93619e52006-03-20 13:44:31 -0500160static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700161{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700162 struct nfs_direct_req *dreq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700163
Trond Myklebust292f3ee2012-04-30 18:31:49 -0400164 dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700165 if (!dreq)
166 return NULL;
167
168 kref_init(&dreq->kref);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400169 kref_get(&dreq->kref);
Trond Myklebustd72b7a62006-03-20 13:44:43 -0500170 init_completion(&dreq->completion);
Fred Isaman1763da12012-04-20 14:47:57 -0400171 INIT_LIST_HEAD(&dreq->mds_cinfo.list);
172 INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);
Chuck Lever15ce4a02006-03-20 13:44:34 -0500173 spin_lock_init(&dreq->lock);
Chuck Lever93619e52006-03-20 13:44:31 -0500174
175 return dreq;
176}
177
Trond Myklebustb4946ff2007-05-30 12:58:00 -0400178static void nfs_direct_req_free(struct kref *kref)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700179{
180 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
Trond Myklebusta8881f52006-03-20 13:44:36 -0500181
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400182 if (dreq->l_ctx != NULL)
183 nfs_put_lock_context(dreq->l_ctx);
Trond Myklebusta8881f52006-03-20 13:44:36 -0500184 if (dreq->ctx != NULL)
185 put_nfs_open_context(dreq->ctx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700186 kmem_cache_free(nfs_direct_cachep, dreq);
187}
188
Trond Myklebustb4946ff2007-05-30 12:58:00 -0400189static void nfs_direct_req_release(struct nfs_direct_req *dreq)
190{
191 kref_put(&dreq->kref, nfs_direct_req_free);
192}
193
Peng Tao62965562012-09-25 14:55:57 +0800194ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq)
195{
196 return dreq->bytes_left;
197}
198EXPORT_SYMBOL_GPL(nfs_dreq_bytes_left);
199
Chuck Leverd4cc9482006-03-20 13:44:28 -0500200/*
Chuck Leverbc0fb202006-03-20 13:44:31 -0500201 * Collects and returns the final error value/byte-count.
202 */
203static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
204{
Chuck Lever15ce4a02006-03-20 13:44:34 -0500205 ssize_t result = -EIOCBQUEUED;
Chuck Leverbc0fb202006-03-20 13:44:31 -0500206
207 /* Async requests don't wait here */
208 if (dreq->iocb)
209 goto out;
210
Matthew Wilcox150030b2007-12-06 16:24:39 -0500211 result = wait_for_completion_killable(&dreq->completion);
Chuck Leverbc0fb202006-03-20 13:44:31 -0500212
213 if (!result)
Chuck Lever15ce4a02006-03-20 13:44:34 -0500214 result = dreq->error;
Chuck Leverbc0fb202006-03-20 13:44:31 -0500215 if (!result)
Chuck Lever15ce4a02006-03-20 13:44:34 -0500216 result = dreq->count;
Chuck Leverbc0fb202006-03-20 13:44:31 -0500217
218out:
Chuck Leverbc0fb202006-03-20 13:44:31 -0500219 return (ssize_t) result;
220}
221
222/*
Trond Myklebust607f31e2006-06-28 16:52:45 -0400223 * Synchronous I/O uses a stack-allocated iocb. Thus we can't trust
224 * the iocb is still valid here if this is a synchronous request.
Chuck Lever63ab46a2006-03-20 13:44:31 -0500225 */
226static void nfs_direct_complete(struct nfs_direct_req *dreq)
227{
Chuck Lever63ab46a2006-03-20 13:44:31 -0500228 if (dreq->iocb) {
Chuck Lever15ce4a02006-03-20 13:44:34 -0500229 long res = (long) dreq->error;
Chuck Lever63ab46a2006-03-20 13:44:31 -0500230 if (!res)
Chuck Lever15ce4a02006-03-20 13:44:34 -0500231 res = (long) dreq->count;
Chuck Lever63ab46a2006-03-20 13:44:31 -0500232 aio_complete(dreq->iocb, res, 0);
Trond Myklebustd72b7a62006-03-20 13:44:43 -0500233 }
234 complete_all(&dreq->completion);
Chuck Lever63ab46a2006-03-20 13:44:31 -0500235
Trond Myklebustb4946ff2007-05-30 12:58:00 -0400236 nfs_direct_req_release(dreq);
Chuck Lever63ab46a2006-03-20 13:44:31 -0500237}
238
Trond Myklebust1385b812012-05-04 13:54:24 -0400239static void nfs_direct_readpage_release(struct nfs_page *req)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240{
Fred Isaman584aa812012-04-20 14:47:51 -0400241 dprintk("NFS: direct read done (%s/%lld %d@%lld)\n",
242 req->wb_context->dentry->d_inode->i_sb->s_id,
243 (long long)NFS_FILEID(req->wb_context->dentry->d_inode),
244 req->wb_bytes,
245 (long long)req_offset(req));
246 nfs_release_request(req);
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400247}
248
Fred Isaman584aa812012-04-20 14:47:51 -0400249static void nfs_direct_read_completion(struct nfs_pgio_header *hdr)
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400250{
Fred Isaman584aa812012-04-20 14:47:51 -0400251 unsigned long bytes = 0;
252 struct nfs_direct_req *dreq = hdr->dreq;
Trond Myklebustfdd1e742008-04-15 16:33:58 -0400253
Fred Isaman584aa812012-04-20 14:47:51 -0400254 if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
255 goto out_put;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700256
Chuck Lever15ce4a02006-03-20 13:44:34 -0500257 spin_lock(&dreq->lock);
Fred Isaman584aa812012-04-20 14:47:51 -0400258 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && (hdr->good_bytes == 0))
259 dreq->error = hdr->error;
260 else
261 dreq->count += hdr->good_bytes;
262 spin_unlock(&dreq->lock);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400263
Trond Myklebust4bd8b012012-05-01 12:49:58 -0400264 while (!list_empty(&hdr->pages)) {
265 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
266 struct page *page = req->wb_page;
Fred Isaman584aa812012-04-20 14:47:51 -0400267
Jeff Laytonbe7e9852012-12-12 12:36:31 -0500268 if (!PageCompound(page) && bytes < hdr->good_bytes)
269 set_page_dirty(page);
Trond Myklebust4bd8b012012-05-01 12:49:58 -0400270 bytes += req->wb_bytes;
271 nfs_list_remove_request(req);
272 nfs_direct_readpage_release(req);
Fred Isaman584aa812012-04-20 14:47:51 -0400273 }
274out_put:
Trond Myklebust607f31e2006-06-28 16:52:45 -0400275 if (put_dreq(dreq))
276 nfs_direct_complete(dreq);
Fred Isaman584aa812012-04-20 14:47:51 -0400277 hdr->release(hdr);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700278}
279
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400280static void nfs_read_sync_pgio_error(struct list_head *head)
Fred Isamancd841602012-04-20 14:47:44 -0400281{
Fred Isaman584aa812012-04-20 14:47:51 -0400282 struct nfs_page *req;
Fred Isamancd841602012-04-20 14:47:44 -0400283
Fred Isaman584aa812012-04-20 14:47:51 -0400284 while (!list_empty(head)) {
285 req = nfs_list_entry(head->next);
286 nfs_list_remove_request(req);
287 nfs_release_request(req);
288 }
Fred Isamancd841602012-04-20 14:47:44 -0400289}
290
Fred Isaman584aa812012-04-20 14:47:51 -0400291static void nfs_direct_pgio_init(struct nfs_pgio_header *hdr)
292{
293 get_dreq(hdr->dreq);
294}
295
296static const struct nfs_pgio_completion_ops nfs_direct_read_completion_ops = {
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400297 .error_cleanup = nfs_read_sync_pgio_error,
Fred Isaman584aa812012-04-20 14:47:51 -0400298 .init_hdr = nfs_direct_pgio_init,
299 .completion = nfs_direct_read_completion,
300};
301
Chuck Leverd4cc9482006-03-20 13:44:28 -0500302/*
Trond Myklebust607f31e2006-06-28 16:52:45 -0400303 * For each rsize'd chunk of the user's buffer, dispatch an NFS READ
304 * operation. If nfs_readdata_alloc() or get_user_pages() fails,
305 * bail and stop sending more reads. Read length accounting is
306 * handled automatically by nfs_direct_read_result(). Otherwise, if
307 * no requests have been sent, just return an error.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700308 */
Fred Isaman584aa812012-04-20 14:47:51 -0400309static ssize_t nfs_direct_read_schedule_segment(struct nfs_pageio_descriptor *desc,
Chuck Lever02fe4942007-11-12 12:17:03 -0500310 const struct iovec *iov,
Mel Gormana564b8f2012-07-31 16:45:12 -0700311 loff_t pos, bool uio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312{
Fred Isaman584aa812012-04-20 14:47:51 -0400313 struct nfs_direct_req *dreq = desc->pg_dreq;
Trond Myklebusta8881f52006-03-20 13:44:36 -0500314 struct nfs_open_context *ctx = dreq->ctx;
Al Viro3d4ff432011-06-22 18:40:12 -0400315 struct inode *inode = ctx->dentry->d_inode;
Chuck Lever02fe4942007-11-12 12:17:03 -0500316 unsigned long user_addr = (unsigned long)iov->iov_base;
317 size_t count = iov->iov_len;
Chuck Lever5dd602f2006-03-20 13:44:29 -0500318 size_t rsize = NFS_SERVER(inode)->rsize;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400319 unsigned int pgbase;
320 int result;
321 ssize_t started = 0;
Fred Isaman584aa812012-04-20 14:47:51 -0400322 struct page **pagevec = NULL;
323 unsigned int npages;
Chuck Lever82b145c2006-06-20 12:57:03 -0400324
Linus Torvalds1da177e2005-04-16 15:20:36 -0700325 do {
Chuck Lever5dd602f2006-03-20 13:44:29 -0500326 size_t bytes;
Fred Isaman584aa812012-04-20 14:47:51 -0400327 int i;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700328
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700329 pgbase = user_addr & ~PAGE_MASK;
Trond Myklebustbf5fc402012-05-04 13:47:16 -0400330 bytes = min(max_t(size_t, rsize, PAGE_SIZE), count);
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700331
Trond Myklebust607f31e2006-06-28 16:52:45 -0400332 result = -ENOMEM;
Fred Isaman584aa812012-04-20 14:47:51 -0400333 npages = nfs_page_array_len(pgbase, bytes);
334 if (!pagevec)
335 pagevec = kmalloc(npages * sizeof(struct page *),
336 GFP_KERNEL);
337 if (!pagevec)
Trond Myklebust607f31e2006-06-28 16:52:45 -0400338 break;
Mel Gormana564b8f2012-07-31 16:45:12 -0700339 if (uio) {
340 down_read(&current->mm->mmap_sem);
341 result = get_user_pages(current, current->mm, user_addr,
Fred Isaman584aa812012-04-20 14:47:51 -0400342 npages, 1, 0, pagevec, NULL);
Mel Gormana564b8f2012-07-31 16:45:12 -0700343 up_read(&current->mm->mmap_sem);
344 if (result < 0)
345 break;
346 } else {
347 WARN_ON(npages != 1);
348 result = get_kernel_page(user_addr, 1, pagevec);
349 if (WARN_ON(result != 1))
350 break;
351 }
352
Fred Isaman584aa812012-04-20 14:47:51 -0400353 if ((unsigned)result < npages) {
Trond Myklebustd9df8d62007-05-22 10:22:20 -0400354 bytes = result * PAGE_SIZE;
355 if (bytes <= pgbase) {
Fred Isaman584aa812012-04-20 14:47:51 -0400356 nfs_direct_release_pages(pagevec, result);
Trond Myklebustd9df8d62007-05-22 10:22:20 -0400357 break;
358 }
359 bytes -= pgbase;
Fred Isaman584aa812012-04-20 14:47:51 -0400360 npages = result;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400361 }
Chuck Lever06cf6f22006-06-20 12:56:49 -0400362
Fred Isaman584aa812012-04-20 14:47:51 -0400363 for (i = 0; i < npages; i++) {
364 struct nfs_page *req;
Trond Myklebustbf5fc402012-05-04 13:47:16 -0400365 unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
Fred Isaman584aa812012-04-20 14:47:51 -0400366 /* XXX do we need to do the eof zeroing found in async_filler? */
367 req = nfs_create_request(dreq->ctx, dreq->inode,
368 pagevec[i],
369 pgbase, req_len);
370 if (IS_ERR(req)) {
Fred Isaman584aa812012-04-20 14:47:51 -0400371 result = PTR_ERR(req);
372 break;
373 }
374 req->wb_index = pos >> PAGE_SHIFT;
375 req->wb_offset = pos & ~PAGE_MASK;
376 if (!nfs_pageio_add_request(desc, req)) {
377 result = desc->pg_error;
378 nfs_release_request(req);
Fred Isaman584aa812012-04-20 14:47:51 -0400379 break;
380 }
381 pgbase = 0;
382 bytes -= req_len;
383 started += req_len;
384 user_addr += req_len;
385 pos += req_len;
386 count -= req_len;
Peng Tao35754bc2012-09-25 14:55:57 +0800387 dreq->bytes_left -= req_len;
Fred Isaman584aa812012-04-20 14:47:51 -0400388 }
Trond Myklebust6d747432012-04-30 13:27:31 -0400389 /* The nfs_page now hold references to these pages */
390 nfs_direct_release_pages(pagevec, npages);
Trond Myklebust71e8cc02012-04-30 13:22:54 -0400391 } while (count != 0 && result >= 0);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400392
Fred Isaman584aa812012-04-20 14:47:51 -0400393 kfree(pagevec);
394
Trond Myklebust607f31e2006-06-28 16:52:45 -0400395 if (started)
Chuck Leverc216fd72007-11-12 12:16:52 -0500396 return started;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400397 return result < 0 ? (ssize_t) result : -EFAULT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700398}
399
Chuck Lever19f73782007-11-12 12:16:47 -0500400static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
401 const struct iovec *iov,
402 unsigned long nr_segs,
Mel Gormana564b8f2012-07-31 16:45:12 -0700403 loff_t pos, bool uio)
Chuck Lever19f73782007-11-12 12:16:47 -0500404{
Fred Isaman584aa812012-04-20 14:47:51 -0400405 struct nfs_pageio_descriptor desc;
Chuck Lever19f73782007-11-12 12:16:47 -0500406 ssize_t result = -EINVAL;
407 size_t requested_bytes = 0;
408 unsigned long seg;
409
Fred Isaman59948db2012-07-18 14:20:49 -0400410 NFS_PROTO(dreq->inode)->read_pageio_init(&desc, dreq->inode,
Fred Isaman584aa812012-04-20 14:47:51 -0400411 &nfs_direct_read_completion_ops);
Chuck Lever19f73782007-11-12 12:16:47 -0500412 get_dreq(dreq);
Fred Isaman584aa812012-04-20 14:47:51 -0400413 desc.pg_dreq = dreq;
Chuck Lever19f73782007-11-12 12:16:47 -0500414
415 for (seg = 0; seg < nr_segs; seg++) {
416 const struct iovec *vec = &iov[seg];
Mel Gormana564b8f2012-07-31 16:45:12 -0700417 result = nfs_direct_read_schedule_segment(&desc, vec, pos, uio);
Chuck Lever19f73782007-11-12 12:16:47 -0500418 if (result < 0)
419 break;
420 requested_bytes += result;
421 if ((size_t)result < vec->iov_len)
422 break;
423 pos += vec->iov_len;
424 }
425
Fred Isaman584aa812012-04-20 14:47:51 -0400426 nfs_pageio_complete(&desc);
427
Chuck Lever839f7ad2011-01-21 15:54:57 +0000428 /*
429 * If no bytes were started, return the error, and let the
430 * generic layer handle the completion.
431 */
432 if (requested_bytes == 0) {
433 nfs_direct_req_release(dreq);
434 return result < 0 ? result : -EIO;
435 }
436
Chuck Lever19f73782007-11-12 12:16:47 -0500437 if (put_dreq(dreq))
438 nfs_direct_complete(dreq);
Chuck Lever839f7ad2011-01-21 15:54:57 +0000439 return 0;
Chuck Lever19f73782007-11-12 12:16:47 -0500440}
441
Chuck Leverc216fd72007-11-12 12:16:52 -0500442static ssize_t nfs_direct_read(struct kiocb *iocb, const struct iovec *iov,
Mel Gormana564b8f2012-07-31 16:45:12 -0700443 unsigned long nr_segs, loff_t pos, bool uio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700444{
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400445 ssize_t result = -ENOMEM;
Chuck Lever99514f82006-03-20 13:44:30 -0500446 struct inode *inode = iocb->ki_filp->f_mapping->host;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700447 struct nfs_direct_req *dreq;
Trond Myklebustb3c54de2012-08-13 17:15:50 -0400448 struct nfs_lock_context *l_ctx;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700449
Trond Myklebust607f31e2006-06-28 16:52:45 -0400450 dreq = nfs_direct_req_alloc();
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400451 if (dreq == NULL)
452 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700453
Chuck Lever91d5b472006-03-20 13:44:14 -0500454 dreq->inode = inode;
Peng Tao35754bc2012-09-25 14:55:57 +0800455 dreq->bytes_left = iov_length(iov, nr_segs);
Trond Myklebustcd3758e2007-08-10 17:44:32 -0400456 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
Trond Myklebustb3c54de2012-08-13 17:15:50 -0400457 l_ctx = nfs_get_lock_context(dreq->ctx);
458 if (IS_ERR(l_ctx)) {
459 result = PTR_ERR(l_ctx);
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400460 goto out_release;
Trond Myklebustb3c54de2012-08-13 17:15:50 -0400461 }
462 dreq->l_ctx = l_ctx;
Chuck Lever487b8372006-03-20 13:44:30 -0500463 if (!is_sync_kiocb(iocb))
464 dreq->iocb = iocb;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700465
Peng Tao7acdb022012-08-24 00:27:48 +0800466 NFS_I(inode)->read_io += iov_length(iov, nr_segs);
Mel Gormana564b8f2012-07-31 16:45:12 -0700467 result = nfs_direct_read_schedule_iovec(dreq, iov, nr_segs, pos, uio);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400468 if (!result)
469 result = nfs_direct_wait(dreq);
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400470out_release:
Trond Myklebustb4946ff2007-05-30 12:58:00 -0400471 nfs_direct_req_release(dreq);
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400472out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700473 return result;
474}
475
Trond Myklebust1d59d612012-05-31 12:22:33 -0400476static void nfs_inode_dio_write_done(struct inode *inode)
477{
478 nfs_zap_mapping(inode, inode->i_mapping);
479 inode_dio_done(inode);
480}
481
Bryan Schumaker89d77c82012-07-30 16:05:25 -0400482#if IS_ENABLED(CONFIG_NFS_V3) || IS_ENABLED(CONFIG_NFS_V4)
Trond Myklebustfad61492006-03-20 13:44:36 -0500483static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700484{
Fred Isaman1763da12012-04-20 14:47:57 -0400485 struct nfs_pageio_descriptor desc;
486 struct nfs_page *req, *tmp;
487 LIST_HEAD(reqs);
488 struct nfs_commit_info cinfo;
489 LIST_HEAD(failed);
490
491 nfs_init_cinfo_from_dreq(&cinfo, dreq);
492 pnfs_recover_commit_reqs(dreq->inode, &reqs, &cinfo);
493 spin_lock(cinfo.lock);
494 nfs_scan_commit_list(&cinfo.mds->list, &reqs, &cinfo, 0);
495 spin_unlock(cinfo.lock);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700496
Trond Myklebustfad61492006-03-20 13:44:36 -0500497 dreq->count = 0;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400498 get_dreq(dreq);
Trond Myklebustfad61492006-03-20 13:44:36 -0500499
Fred Isamanc95908e2012-07-18 14:20:50 -0400500 NFS_PROTO(dreq->inode)->write_pageio_init(&desc, dreq->inode, FLUSH_STABLE,
Fred Isaman1763da12012-04-20 14:47:57 -0400501 &nfs_direct_write_completion_ops);
502 desc.pg_dreq = dreq;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400503
Fred Isaman1763da12012-04-20 14:47:57 -0400504 list_for_each_entry_safe(req, tmp, &reqs, wb_list) {
505 if (!nfs_pageio_add_request(&desc, req)) {
Trond Myklebust4035c242012-07-08 10:24:10 -0400506 nfs_list_remove_request(req);
Fred Isaman1763da12012-04-20 14:47:57 -0400507 nfs_list_add_request(req, &failed);
508 spin_lock(cinfo.lock);
509 dreq->flags = 0;
510 dreq->error = -EIO;
511 spin_unlock(cinfo.lock);
512 }
Trond Myklebust5a695da2012-06-19 13:39:14 -0400513 nfs_release_request(req);
Fred Isaman1763da12012-04-20 14:47:57 -0400514 }
515 nfs_pageio_complete(&desc);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400516
Trond Myklebust4035c242012-07-08 10:24:10 -0400517 while (!list_empty(&failed)) {
518 req = nfs_list_entry(failed.next);
519 nfs_list_remove_request(req);
Trond Myklebust1d1afcb2012-05-09 14:04:55 -0400520 nfs_unlock_and_release_request(req);
Trond Myklebust4035c242012-07-08 10:24:10 -0400521 }
Trond Myklebust607f31e2006-06-28 16:52:45 -0400522
523 if (put_dreq(dreq))
Fred Isaman1763da12012-04-20 14:47:57 -0400524 nfs_direct_write_complete(dreq, dreq->inode);
Trond Myklebustfad61492006-03-20 13:44:36 -0500525}
526
Fred Isaman1763da12012-04-20 14:47:57 -0400527static void nfs_direct_commit_complete(struct nfs_commit_data *data)
Trond Myklebustfad61492006-03-20 13:44:36 -0500528{
Fred Isaman0b7c0152012-04-20 14:47:39 -0400529 struct nfs_direct_req *dreq = data->dreq;
Fred Isaman1763da12012-04-20 14:47:57 -0400530 struct nfs_commit_info cinfo;
531 struct nfs_page *req;
Trond Myklebustc9d8f892008-04-15 16:56:39 -0400532 int status = data->task.tk_status;
533
Fred Isaman1763da12012-04-20 14:47:57 -0400534 nfs_init_cinfo_from_dreq(&cinfo, dreq);
Trond Myklebustc9d8f892008-04-15 16:56:39 -0400535 if (status < 0) {
Trond Myklebust60fa3f72007-04-14 19:11:52 -0400536 dprintk("NFS: %5u commit failed with error %d.\n",
Fred Isaman1763da12012-04-20 14:47:57 -0400537 data->task.tk_pid, status);
Trond Myklebustfad61492006-03-20 13:44:36 -0500538 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
Trond Myklebust60fa3f72007-04-14 19:11:52 -0400539 } else if (memcmp(&dreq->verf, &data->verf, sizeof(data->verf))) {
Trond Myklebustc9d8f892008-04-15 16:56:39 -0400540 dprintk("NFS: %5u commit verify failed\n", data->task.tk_pid);
Trond Myklebustfad61492006-03-20 13:44:36 -0500541 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
542 }
543
Trond Myklebustc9d8f892008-04-15 16:56:39 -0400544 dprintk("NFS: %5u commit returned %d\n", data->task.tk_pid, status);
Fred Isaman1763da12012-04-20 14:47:57 -0400545 while (!list_empty(&data->pages)) {
546 req = nfs_list_entry(data->pages.next);
547 nfs_list_remove_request(req);
548 if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) {
549 /* Note the rewrite will go through mds */
550 nfs_mark_request_commit(req, NULL, &cinfo);
Fred Isaman906369e2012-06-08 16:48:33 -0400551 } else
552 nfs_release_request(req);
Trond Myklebust1d1afcb2012-05-09 14:04:55 -0400553 nfs_unlock_and_release_request(req);
Fred Isaman1763da12012-04-20 14:47:57 -0400554 }
555
556 if (atomic_dec_and_test(&cinfo.mds->rpcs_out))
557 nfs_direct_write_complete(dreq, data->inode);
Trond Myklebustfad61492006-03-20 13:44:36 -0500558}
559
Fred Isaman1763da12012-04-20 14:47:57 -0400560static void nfs_direct_error_cleanup(struct nfs_inode *nfsi)
561{
562 /* There is no lock to clear */
563}
564
565static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops = {
566 .completion = nfs_direct_commit_complete,
567 .error_cleanup = nfs_direct_error_cleanup,
Trond Myklebustfad61492006-03-20 13:44:36 -0500568};
569
570static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
571{
Fred Isaman1763da12012-04-20 14:47:57 -0400572 int res;
573 struct nfs_commit_info cinfo;
574 LIST_HEAD(mds_list);
Trond Myklebustfad61492006-03-20 13:44:36 -0500575
Fred Isaman1763da12012-04-20 14:47:57 -0400576 nfs_init_cinfo_from_dreq(&cinfo, dreq);
577 nfs_scan_commit(dreq->inode, &mds_list, &cinfo);
578 res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);
579 if (res < 0) /* res == -ENOMEM */
580 nfs_direct_write_reschedule(dreq);
Trond Myklebustfad61492006-03-20 13:44:36 -0500581}
582
Fred Isaman1763da12012-04-20 14:47:57 -0400583static void nfs_direct_write_schedule_work(struct work_struct *work)
Trond Myklebustfad61492006-03-20 13:44:36 -0500584{
Fred Isaman1763da12012-04-20 14:47:57 -0400585 struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);
Trond Myklebustfad61492006-03-20 13:44:36 -0500586 int flags = dreq->flags;
587
588 dreq->flags = 0;
589 switch (flags) {
590 case NFS_ODIRECT_DO_COMMIT:
591 nfs_direct_commit_schedule(dreq);
592 break;
593 case NFS_ODIRECT_RESCHED_WRITES:
594 nfs_direct_write_reschedule(dreq);
595 break;
596 default:
Trond Myklebust1d59d612012-05-31 12:22:33 -0400597 nfs_inode_dio_write_done(dreq->inode);
Trond Myklebustfad61492006-03-20 13:44:36 -0500598 nfs_direct_complete(dreq);
599 }
600}
601
Fred Isaman1763da12012-04-20 14:47:57 -0400602static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
Trond Myklebustfad61492006-03-20 13:44:36 -0500603{
Fred Isaman1763da12012-04-20 14:47:57 -0400604 schedule_work(&dreq->work); /* Calls nfs_direct_write_schedule_work */
Trond Myklebustfad61492006-03-20 13:44:36 -0500605}
Fred Isaman1763da12012-04-20 14:47:57 -0400606
Trond Myklebustfad61492006-03-20 13:44:36 -0500607#else
Bryan Schumaker24fc9212012-04-30 13:27:11 -0400608static void nfs_direct_write_schedule_work(struct work_struct *work)
609{
610}
Trond Myklebustfad61492006-03-20 13:44:36 -0500611
612static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
613{
Trond Myklebust1d59d612012-05-31 12:22:33 -0400614 nfs_inode_dio_write_done(inode);
Trond Myklebustfad61492006-03-20 13:44:36 -0500615 nfs_direct_complete(dreq);
616}
617#endif
618
Trond Myklebustc9d8f892008-04-15 16:56:39 -0400619/*
620 * NB: Return the value of the first error return code. Subsequent
621 * errors after the first one are ignored.
622 */
Chuck Lever462d5b32006-03-20 13:44:32 -0500623/*
Trond Myklebust607f31e2006-06-28 16:52:45 -0400624 * For each wsize'd chunk of the user's buffer, dispatch an NFS WRITE
625 * operation. If nfs_writedata_alloc() or get_user_pages() fails,
626 * bail and stop sending more writes. Write length accounting is
627 * handled automatically by nfs_direct_write_result(). Otherwise, if
628 * no requests have been sent, just return an error.
Chuck Lever462d5b32006-03-20 13:44:32 -0500629 */
Fred Isaman1763da12012-04-20 14:47:57 -0400630static ssize_t nfs_direct_write_schedule_segment(struct nfs_pageio_descriptor *desc,
Chuck Lever02fe4942007-11-12 12:17:03 -0500631 const struct iovec *iov,
Mel Gormana564b8f2012-07-31 16:45:12 -0700632 loff_t pos, bool uio)
Chuck Lever462d5b32006-03-20 13:44:32 -0500633{
Fred Isaman1763da12012-04-20 14:47:57 -0400634 struct nfs_direct_req *dreq = desc->pg_dreq;
Trond Myklebusta8881f52006-03-20 13:44:36 -0500635 struct nfs_open_context *ctx = dreq->ctx;
Al Viro3d4ff432011-06-22 18:40:12 -0400636 struct inode *inode = ctx->dentry->d_inode;
Chuck Lever02fe4942007-11-12 12:17:03 -0500637 unsigned long user_addr = (unsigned long)iov->iov_base;
638 size_t count = iov->iov_len;
Chuck Lever462d5b32006-03-20 13:44:32 -0500639 size_t wsize = NFS_SERVER(inode)->wsize;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400640 unsigned int pgbase;
641 int result;
642 ssize_t started = 0;
Fred Isaman1763da12012-04-20 14:47:57 -0400643 struct page **pagevec = NULL;
644 unsigned int npages;
Chuck Lever82b145c2006-06-20 12:57:03 -0400645
Chuck Lever462d5b32006-03-20 13:44:32 -0500646 do {
Chuck Lever462d5b32006-03-20 13:44:32 -0500647 size_t bytes;
Fred Isaman1763da12012-04-20 14:47:57 -0400648 int i;
Chuck Lever462d5b32006-03-20 13:44:32 -0500649
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700650 pgbase = user_addr & ~PAGE_MASK;
Trond Myklebustbf5fc402012-05-04 13:47:16 -0400651 bytes = min(max_t(size_t, wsize, PAGE_SIZE), count);
Trond Myklebuste9f7bee2006-09-08 09:48:54 -0700652
Trond Myklebust607f31e2006-06-28 16:52:45 -0400653 result = -ENOMEM;
Fred Isaman1763da12012-04-20 14:47:57 -0400654 npages = nfs_page_array_len(pgbase, bytes);
655 if (!pagevec)
656 pagevec = kmalloc(npages * sizeof(struct page *), GFP_KERNEL);
657 if (!pagevec)
Trond Myklebust607f31e2006-06-28 16:52:45 -0400658 break;
659
Mel Gormana564b8f2012-07-31 16:45:12 -0700660 if (uio) {
661 down_read(&current->mm->mmap_sem);
662 result = get_user_pages(current, current->mm, user_addr,
663 npages, 0, 0, pagevec, NULL);
664 up_read(&current->mm->mmap_sem);
665 if (result < 0)
666 break;
667 } else {
668 WARN_ON(npages != 1);
669 result = get_kernel_page(user_addr, 0, pagevec);
670 if (WARN_ON(result != 1))
671 break;
672 }
Fred Isaman1763da12012-04-20 14:47:57 -0400673
674 if ((unsigned)result < npages) {
Trond Myklebustd9df8d62007-05-22 10:22:20 -0400675 bytes = result * PAGE_SIZE;
676 if (bytes <= pgbase) {
Fred Isaman1763da12012-04-20 14:47:57 -0400677 nfs_direct_release_pages(pagevec, result);
Trond Myklebustd9df8d62007-05-22 10:22:20 -0400678 break;
679 }
680 bytes -= pgbase;
Fred Isaman1763da12012-04-20 14:47:57 -0400681 npages = result;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400682 }
683
Fred Isaman1763da12012-04-20 14:47:57 -0400684 for (i = 0; i < npages; i++) {
685 struct nfs_page *req;
Trond Myklebustbf5fc402012-05-04 13:47:16 -0400686 unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400687
Fred Isaman1763da12012-04-20 14:47:57 -0400688 req = nfs_create_request(dreq->ctx, dreq->inode,
689 pagevec[i],
690 pgbase, req_len);
691 if (IS_ERR(req)) {
Fred Isaman1763da12012-04-20 14:47:57 -0400692 result = PTR_ERR(req);
693 break;
694 }
695 nfs_lock_request(req);
696 req->wb_index = pos >> PAGE_SHIFT;
697 req->wb_offset = pos & ~PAGE_MASK;
698 if (!nfs_pageio_add_request(desc, req)) {
699 result = desc->pg_error;
Trond Myklebust1d1afcb2012-05-09 14:04:55 -0400700 nfs_unlock_and_release_request(req);
Trond Myklebust71e8cc02012-04-30 13:22:54 -0400701 break;
Fred Isaman1763da12012-04-20 14:47:57 -0400702 }
703 pgbase = 0;
704 bytes -= req_len;
705 started += req_len;
706 user_addr += req_len;
707 pos += req_len;
708 count -= req_len;
Peng Tao35754bc2012-09-25 14:55:57 +0800709 dreq->bytes_left -= req_len;
Fred Isaman1763da12012-04-20 14:47:57 -0400710 }
Trond Myklebust6d747432012-04-30 13:27:31 -0400711 /* The nfs_page now hold references to these pages */
712 nfs_direct_release_pages(pagevec, npages);
Trond Myklebust71e8cc02012-04-30 13:22:54 -0400713 } while (count != 0 && result >= 0);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400714
Fred Isaman1763da12012-04-20 14:47:57 -0400715 kfree(pagevec);
716
Trond Myklebust607f31e2006-06-28 16:52:45 -0400717 if (started)
Chuck Leverc216fd72007-11-12 12:16:52 -0500718 return started;
Trond Myklebust607f31e2006-06-28 16:52:45 -0400719 return result < 0 ? (ssize_t) result : -EFAULT;
Chuck Lever462d5b32006-03-20 13:44:32 -0500720}
721
Fred Isaman1763da12012-04-20 14:47:57 -0400722static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
723{
724 struct nfs_direct_req *dreq = hdr->dreq;
725 struct nfs_commit_info cinfo;
726 int bit = -1;
727 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
728
729 if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
730 goto out_put;
731
732 nfs_init_cinfo_from_dreq(&cinfo, dreq);
733
734 spin_lock(&dreq->lock);
735
736 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags)) {
737 dreq->flags = 0;
738 dreq->error = hdr->error;
739 }
740 if (dreq->error != 0)
741 bit = NFS_IOHDR_ERROR;
742 else {
743 dreq->count += hdr->good_bytes;
744 if (test_bit(NFS_IOHDR_NEED_RESCHED, &hdr->flags)) {
745 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
746 bit = NFS_IOHDR_NEED_RESCHED;
747 } else if (test_bit(NFS_IOHDR_NEED_COMMIT, &hdr->flags)) {
748 if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES)
749 bit = NFS_IOHDR_NEED_RESCHED;
750 else if (dreq->flags == 0) {
Trond Myklebust9bce0082012-06-05 18:32:03 -0400751 memcpy(&dreq->verf, hdr->verf,
Fred Isaman1763da12012-04-20 14:47:57 -0400752 sizeof(dreq->verf));
753 bit = NFS_IOHDR_NEED_COMMIT;
754 dreq->flags = NFS_ODIRECT_DO_COMMIT;
755 } else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) {
Trond Myklebust9bce0082012-06-05 18:32:03 -0400756 if (memcmp(&dreq->verf, hdr->verf, sizeof(dreq->verf))) {
Fred Isaman1763da12012-04-20 14:47:57 -0400757 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
758 bit = NFS_IOHDR_NEED_RESCHED;
759 } else
760 bit = NFS_IOHDR_NEED_COMMIT;
761 }
762 }
763 }
764 spin_unlock(&dreq->lock);
765
766 while (!list_empty(&hdr->pages)) {
767 req = nfs_list_entry(hdr->pages.next);
768 nfs_list_remove_request(req);
769 switch (bit) {
770 case NFS_IOHDR_NEED_RESCHED:
771 case NFS_IOHDR_NEED_COMMIT:
Trond Myklebust04277082012-05-09 13:54:53 -0400772 kref_get(&req->wb_kref);
Fred Isaman1763da12012-04-20 14:47:57 -0400773 nfs_mark_request_commit(req, hdr->lseg, &cinfo);
Fred Isaman1763da12012-04-20 14:47:57 -0400774 }
Trond Myklebust1d1afcb2012-05-09 14:04:55 -0400775 nfs_unlock_and_release_request(req);
Fred Isaman1763da12012-04-20 14:47:57 -0400776 }
777
778out_put:
779 if (put_dreq(dreq))
780 nfs_direct_write_complete(dreq, hdr->inode);
781 hdr->release(hdr);
782}
783
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400784static void nfs_write_sync_pgio_error(struct list_head *head)
785{
786 struct nfs_page *req;
787
788 while (!list_empty(head)) {
789 req = nfs_list_entry(head->next);
790 nfs_list_remove_request(req);
Trond Myklebust1d1afcb2012-05-09 14:04:55 -0400791 nfs_unlock_and_release_request(req);
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400792 }
793}
794
Fred Isaman1763da12012-04-20 14:47:57 -0400795static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops = {
Trond Myklebust3e9e0ca32012-04-30 13:40:06 -0400796 .error_cleanup = nfs_write_sync_pgio_error,
Fred Isaman1763da12012-04-20 14:47:57 -0400797 .init_hdr = nfs_direct_pgio_init,
798 .completion = nfs_direct_write_completion,
799};
800
Chuck Lever19f73782007-11-12 12:16:47 -0500801static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
802 const struct iovec *iov,
803 unsigned long nr_segs,
Mel Gormana564b8f2012-07-31 16:45:12 -0700804 loff_t pos, bool uio)
Chuck Lever19f73782007-11-12 12:16:47 -0500805{
Fred Isaman1763da12012-04-20 14:47:57 -0400806 struct nfs_pageio_descriptor desc;
Trond Myklebust1d59d612012-05-31 12:22:33 -0400807 struct inode *inode = dreq->inode;
Chuck Lever19f73782007-11-12 12:16:47 -0500808 ssize_t result = 0;
809 size_t requested_bytes = 0;
810 unsigned long seg;
811
Fred Isamanc95908e2012-07-18 14:20:50 -0400812 NFS_PROTO(inode)->write_pageio_init(&desc, inode, FLUSH_COND_STABLE,
Fred Isaman1763da12012-04-20 14:47:57 -0400813 &nfs_direct_write_completion_ops);
814 desc.pg_dreq = dreq;
Chuck Lever19f73782007-11-12 12:16:47 -0500815 get_dreq(dreq);
Trond Myklebust1d59d612012-05-31 12:22:33 -0400816 atomic_inc(&inode->i_dio_count);
Chuck Lever19f73782007-11-12 12:16:47 -0500817
Peng Tao7acdb022012-08-24 00:27:48 +0800818 NFS_I(dreq->inode)->write_io += iov_length(iov, nr_segs);
Chuck Lever19f73782007-11-12 12:16:47 -0500819 for (seg = 0; seg < nr_segs; seg++) {
820 const struct iovec *vec = &iov[seg];
Mel Gormana564b8f2012-07-31 16:45:12 -0700821 result = nfs_direct_write_schedule_segment(&desc, vec, pos, uio);
Chuck Lever19f73782007-11-12 12:16:47 -0500822 if (result < 0)
823 break;
824 requested_bytes += result;
825 if ((size_t)result < vec->iov_len)
826 break;
827 pos += vec->iov_len;
828 }
Fred Isaman1763da12012-04-20 14:47:57 -0400829 nfs_pageio_complete(&desc);
Chuck Lever19f73782007-11-12 12:16:47 -0500830
Chuck Lever839f7ad2011-01-21 15:54:57 +0000831 /*
832 * If no bytes were started, return the error, and let the
833 * generic layer handle the completion.
834 */
835 if (requested_bytes == 0) {
Trond Myklebust1d59d612012-05-31 12:22:33 -0400836 inode_dio_done(inode);
Chuck Lever839f7ad2011-01-21 15:54:57 +0000837 nfs_direct_req_release(dreq);
838 return result < 0 ? result : -EIO;
839 }
840
Chuck Lever19f73782007-11-12 12:16:47 -0500841 if (put_dreq(dreq))
842 nfs_direct_write_complete(dreq, dreq->inode);
Chuck Lever839f7ad2011-01-21 15:54:57 +0000843 return 0;
Chuck Lever19f73782007-11-12 12:16:47 -0500844}
845
Chuck Leverc216fd72007-11-12 12:16:52 -0500846static ssize_t nfs_direct_write(struct kiocb *iocb, const struct iovec *iov,
847 unsigned long nr_segs, loff_t pos,
Mel Gormana564b8f2012-07-31 16:45:12 -0700848 size_t count, bool uio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700849{
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400850 ssize_t result = -ENOMEM;
Chuck Leverc89f2ee2006-03-20 13:44:33 -0500851 struct inode *inode = iocb->ki_filp->f_mapping->host;
Chuck Lever462d5b32006-03-20 13:44:32 -0500852 struct nfs_direct_req *dreq;
Trond Myklebustb3c54de2012-08-13 17:15:50 -0400853 struct nfs_lock_context *l_ctx;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700854
Trond Myklebust607f31e2006-06-28 16:52:45 -0400855 dreq = nfs_direct_req_alloc();
Chuck Lever462d5b32006-03-20 13:44:32 -0500856 if (!dreq)
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400857 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700858
Chuck Leverc89f2ee2006-03-20 13:44:33 -0500859 dreq->inode = inode;
Peng Tao35754bc2012-09-25 14:55:57 +0800860 dreq->bytes_left = count;
Trond Myklebustcd3758e2007-08-10 17:44:32 -0400861 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
Trond Myklebustb3c54de2012-08-13 17:15:50 -0400862 l_ctx = nfs_get_lock_context(dreq->ctx);
863 if (IS_ERR(l_ctx)) {
864 result = PTR_ERR(l_ctx);
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400865 goto out_release;
Trond Myklebustb3c54de2012-08-13 17:15:50 -0400866 }
867 dreq->l_ctx = l_ctx;
Chuck Leverc89f2ee2006-03-20 13:44:33 -0500868 if (!is_sync_kiocb(iocb))
869 dreq->iocb = iocb;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700870
Mel Gormana564b8f2012-07-31 16:45:12 -0700871 result = nfs_direct_write_schedule_iovec(dreq, iov, nr_segs, pos, uio);
Trond Myklebust607f31e2006-06-28 16:52:45 -0400872 if (!result)
873 result = nfs_direct_wait(dreq);
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400874out_release:
Trond Myklebustb4946ff2007-05-30 12:58:00 -0400875 nfs_direct_req_release(dreq);
Trond Myklebustf11ac8d2010-06-25 16:35:53 -0400876out:
Linus Torvalds1da177e2005-04-16 15:20:36 -0700877 return result;
878}
879
880/**
881 * nfs_file_direct_read - file direct read operation for NFS files
882 * @iocb: target I/O control block
Badari Pulavarty027445c2006-09-30 23:28:46 -0700883 * @iov: vector of user buffers into which to read data
884 * @nr_segs: size of iov vector
Chuck Lever88467052006-03-20 13:44:34 -0500885 * @pos: byte offset in file where reading starts
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886 *
887 * We use this function for direct reads instead of calling
888 * generic_file_aio_read() in order to avoid gfar's check to see if
889 * the request starts before the end of the file. For that check
890 * to work, we must generate a GETATTR before each direct read, and
891 * even then there is a window between the GETATTR and the subsequent
Chuck Lever88467052006-03-20 13:44:34 -0500892 * READ where the file size could change. Our preference is simply
Linus Torvalds1da177e2005-04-16 15:20:36 -0700893 * to do all reads the application wants, and the server will take
894 * care of managing the end of file boundary.
Chuck Lever88467052006-03-20 13:44:34 -0500895 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896 * This function also eliminates unnecessarily updating the file's
897 * atime locally, as the NFS server sets the file's atime, and this
898 * client must read the updated atime from the server back into its
899 * cache.
900 */
Badari Pulavarty027445c2006-09-30 23:28:46 -0700901ssize_t nfs_file_direct_read(struct kiocb *iocb, const struct iovec *iov,
Mel Gormana564b8f2012-07-31 16:45:12 -0700902 unsigned long nr_segs, loff_t pos, bool uio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700903{
904 ssize_t retval = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700905 struct file *file = iocb->ki_filp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700906 struct address_space *mapping = file->f_mapping;
Chuck Leverc216fd72007-11-12 12:16:52 -0500907 size_t count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700908
Chuck Leverc216fd72007-11-12 12:16:52 -0500909 count = iov_length(iov, nr_segs);
910 nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count);
911
Chuck Lever6da24bc2008-06-11 17:55:58 -0400912 dfprintk(FILE, "NFS: direct read(%s/%s, %zd@%Ld)\n",
Josef "Jeff" Sipek01cce932006-12-08 02:36:40 -0800913 file->f_path.dentry->d_parent->d_name.name,
914 file->f_path.dentry->d_name.name,
Chuck Leverc216fd72007-11-12 12:16:52 -0500915 count, (long long) pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700916
Linus Torvalds1da177e2005-04-16 15:20:36 -0700917 retval = 0;
918 if (!count)
919 goto out;
920
Trond Myklebust29884df2005-12-13 16:13:54 -0500921 retval = nfs_sync_mapping(mapping);
922 if (retval)
923 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700924
Konstantin Khlebnikov7ec10f22011-02-22 00:28:34 +0300925 task_io_account_read(count);
926
Mel Gormana564b8f2012-07-31 16:45:12 -0700927 retval = nfs_direct_read(iocb, iov, nr_segs, pos, uio);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700928 if (retval > 0)
Chuck Lever0cdd80d2006-03-20 13:44:29 -0500929 iocb->ki_pos = pos + retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700930
931out:
932 return retval;
933}
934
935/**
936 * nfs_file_direct_write - file direct write operation for NFS files
937 * @iocb: target I/O control block
Badari Pulavarty027445c2006-09-30 23:28:46 -0700938 * @iov: vector of user buffers from which to write data
939 * @nr_segs: size of iov vector
Chuck Lever88467052006-03-20 13:44:34 -0500940 * @pos: byte offset in file where writing starts
Linus Torvalds1da177e2005-04-16 15:20:36 -0700941 *
942 * We use this function for direct writes instead of calling
943 * generic_file_aio_write() in order to avoid taking the inode
944 * semaphore and updating the i_size. The NFS server will set
945 * the new i_size and this client must read the updated size
946 * back into its cache. We let the server do generic write
947 * parameter checking and report problems.
948 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700949 * We eliminate local atime updates, see direct read above.
950 *
951 * We avoid unnecessary page cache invalidations for normal cached
952 * readers of this file.
953 *
954 * Note that O_APPEND is not supported for NFS direct writes, as there
955 * is no atomic O_APPEND write facility in the NFS protocol.
956 */
Badari Pulavarty027445c2006-09-30 23:28:46 -0700957ssize_t nfs_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
Mel Gormana564b8f2012-07-31 16:45:12 -0700958 unsigned long nr_segs, loff_t pos, bool uio)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700959{
Chuck Lever070ea602007-05-19 17:22:52 -0400960 ssize_t retval = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700961 struct file *file = iocb->ki_filp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700962 struct address_space *mapping = file->f_mapping;
Chuck Leverc216fd72007-11-12 12:16:52 -0500963 size_t count;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700964
Chuck Leverc216fd72007-11-12 12:16:52 -0500965 count = iov_length(iov, nr_segs);
966 nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count);
967
Chuck Lever6da24bc2008-06-11 17:55:58 -0400968 dfprintk(FILE, "NFS: direct write(%s/%s, %zd@%Ld)\n",
Josef "Jeff" Sipek01cce932006-12-08 02:36:40 -0800969 file->f_path.dentry->d_parent->d_name.name,
970 file->f_path.dentry->d_name.name,
Chuck Leverc216fd72007-11-12 12:16:52 -0500971 count, (long long) pos);
Badari Pulavarty027445c2006-09-30 23:28:46 -0700972
Chuck Leverce1a8e62005-11-30 18:08:17 -0500973 retval = generic_write_checks(file, &pos, &count, 0);
974 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700975 goto out;
Chuck Leverce1a8e62005-11-30 18:08:17 -0500976
977 retval = -EINVAL;
978 if ((ssize_t) count < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700979 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700980 retval = 0;
981 if (!count)
982 goto out;
Chuck Leverce1a8e62005-11-30 18:08:17 -0500983
Trond Myklebust29884df2005-12-13 16:13:54 -0500984 retval = nfs_sync_mapping(mapping);
985 if (retval)
986 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700987
Konstantin Khlebnikov7ec10f22011-02-22 00:28:34 +0300988 task_io_account_write(count);
989
Mel Gormana564b8f2012-07-31 16:45:12 -0700990 retval = nfs_direct_write(iocb, iov, nr_segs, pos, count, uio);
Fred Isaman1763da12012-04-20 14:47:57 -0400991 if (retval > 0) {
992 struct inode *inode = mapping->host;
Chuck Lever9eafa8c2006-03-20 13:44:33 -0500993
Chuck Leverce1a8e62005-11-30 18:08:17 -0500994 iocb->ki_pos = pos + retval;
Fred Isaman1763da12012-04-20 14:47:57 -0400995 spin_lock(&inode->i_lock);
996 if (i_size_read(inode) < iocb->ki_pos)
997 i_size_write(inode, iocb->ki_pos);
998 spin_unlock(&inode->i_lock);
999 }
Linus Torvalds1da177e2005-04-16 15:20:36 -07001000out:
1001 return retval;
1002}
1003
Chuck Lever88467052006-03-20 13:44:34 -05001004/**
1005 * nfs_init_directcache - create a slab cache for nfs_direct_req structures
1006 *
1007 */
David Howellsf7b422b2006-06-09 09:34:33 -04001008int __init nfs_init_directcache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001009{
1010 nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
1011 sizeof(struct nfs_direct_req),
Paul Jacksonfffb60f2006-03-24 03:16:06 -08001012 0, (SLAB_RECLAIM_ACCOUNT|
1013 SLAB_MEM_SPREAD),
Paul Mundt20c2df82007-07-20 10:11:58 +09001014 NULL);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001015 if (nfs_direct_cachep == NULL)
1016 return -ENOMEM;
1017
1018 return 0;
1019}
1020
Chuck Lever88467052006-03-20 13:44:34 -05001021/**
David Howellsf7b422b2006-06-09 09:34:33 -04001022 * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures
Chuck Lever88467052006-03-20 13:44:34 -05001023 *
1024 */
David Brownell266bee82006-06-27 12:59:15 -07001025void nfs_destroy_directcache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -07001026{
Alexey Dobriyan1a1d92c2006-09-27 01:49:40 -07001027 kmem_cache_destroy(nfs_direct_cachep);
Linus Torvalds1da177e2005-04-16 15:20:36 -07001028}