blob: 8ca9707be6c9d06f28a843fbe6282afd1d405e77 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/nfs/direct.c
3 *
4 * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
5 *
6 * High-performance uncached I/O for the Linux NFS client
7 *
8 * There are important applications whose performance or correctness
9 * depends on uncached access to file data. Database clusters
Chuck Lever88467052006-03-20 13:44:34 -050010 * (multiple copies of the same instance running on separate hosts)
Linus Torvalds1da177e2005-04-16 15:20:36 -070011 * implement their own cache coherency protocol that subsumes file
Chuck Lever88467052006-03-20 13:44:34 -050012 * system cache protocols. Applications that process datasets
13 * considerably larger than the client's memory do not always benefit
14 * from a local cache. A streaming video server, for instance, has no
Linus Torvalds1da177e2005-04-16 15:20:36 -070015 * need to cache the contents of a file.
16 *
17 * When an application requests uncached I/O, all read and write requests
18 * are made directly to the server; data stored or fetched via these
19 * requests is not cached in the Linux page cache. The client does not
20 * correct unaligned requests from applications. All requested bytes are
21 * held on permanent storage before a direct write system call returns to
22 * an application.
23 *
24 * Solaris implements an uncached I/O facility called directio() that
25 * is used for backups and sequential I/O to very large files. Solaris
26 * also supports uncaching whole NFS partitions with "-o forcedirectio,"
27 * an undocumented mount option.
28 *
29 * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
30 * help from Andrew Morton.
31 *
32 * 18 Dec 2001 Initial implementation for 2.4 --cel
33 * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy
34 * 08 Jun 2003 Port to 2.5 APIs --cel
35 * 31 Mar 2004 Handle direct I/O without VFS support --cel
36 * 15 Sep 2004 Parallel async reads --cel
Chuck Lever88467052006-03-20 13:44:34 -050037 * 04 May 2005 support O_DIRECT with aio --cel
Linus Torvalds1da177e2005-04-16 15:20:36 -070038 *
39 */
40
41#include <linux/config.h>
42#include <linux/errno.h>
43#include <linux/sched.h>
44#include <linux/kernel.h>
45#include <linux/smp_lock.h>
46#include <linux/file.h>
47#include <linux/pagemap.h>
48#include <linux/kref.h>
49
50#include <linux/nfs_fs.h>
51#include <linux/nfs_page.h>
52#include <linux/sunrpc/clnt.h>
53
54#include <asm/system.h>
55#include <asm/uaccess.h>
56#include <asm/atomic.h>
57
Chuck Lever91d5b472006-03-20 13:44:14 -050058#include "iostat.h"
Linus Torvalds1da177e2005-04-16 15:20:36 -070059
Linus Torvalds1da177e2005-04-16 15:20:36 -070060#define NFSDBG_FACILITY NFSDBG_VFS
Linus Torvalds1da177e2005-04-16 15:20:36 -070061
62static kmem_cache_t *nfs_direct_cachep;
63
64/*
65 * This represents a set of asynchronous requests that we're waiting on
66 */
67struct nfs_direct_req {
68 struct kref kref; /* release manager */
Chuck Lever15ce4a02006-03-20 13:44:34 -050069
70 /* I/O parameters */
Trond Myklebustccf01ef2006-06-25 06:27:31 -040071 struct list_head list, /* nfs_read/write_data structs */
72 rewrite_list; /* saved nfs_write_data structs */
Trond Myklebusta8881f52006-03-20 13:44:36 -050073 struct nfs_open_context *ctx; /* file open context info */
Chuck Lever99514f82006-03-20 13:44:30 -050074 struct kiocb * iocb; /* controlling i/o request */
Chuck Lever88467052006-03-20 13:44:34 -050075 struct inode * inode; /* target file of i/o */
Trond Myklebustccf01ef2006-06-25 06:27:31 -040076 unsigned long user_addr; /* location of user's buffer */
77 size_t user_count; /* total bytes to move */
78 loff_t pos; /* starting offset in file */
79 struct page ** pages; /* pages in our buffer */
80 unsigned int npages; /* count of pages */
Chuck Lever15ce4a02006-03-20 13:44:34 -050081
82 /* completion state */
83 spinlock_t lock; /* protect completion state */
Trond Myklebustccf01ef2006-06-25 06:27:31 -040084 int outstanding; /* i/os we're waiting for */
Chuck Lever15ce4a02006-03-20 13:44:34 -050085 ssize_t count, /* bytes actually processed */
Linus Torvalds1da177e2005-04-16 15:20:36 -070086 error; /* any reported error */
Trond Myklebustd72b7a62006-03-20 13:44:43 -050087 struct completion completion; /* wait for i/o completion */
Trond Myklebustfad61492006-03-20 13:44:36 -050088
89 /* commit state */
90 struct nfs_write_data * commit_data; /* special write_data for commits */
91 int flags;
92#define NFS_ODIRECT_DO_COMMIT (1) /* an unstable reply was received */
93#define NFS_ODIRECT_RESCHED_WRITES (2) /* write verification failed */
94 struct nfs_writeverf verf; /* unstable write verifier */
Linus Torvalds1da177e2005-04-16 15:20:36 -070095};
96
Trond Myklebustccf01ef2006-06-25 06:27:31 -040097static void nfs_direct_write_schedule(struct nfs_direct_req *dreq, int sync);
Trond Myklebustfad61492006-03-20 13:44:36 -050098static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode);
Chuck Lever82b145c2006-06-20 12:57:03 -040099
Linus Torvalds1da177e2005-04-16 15:20:36 -0700100/**
Chuck Leverb8a32e22006-03-20 13:44:28 -0500101 * nfs_direct_IO - NFS address space operation for direct I/O
102 * @rw: direction (read or write)
103 * @iocb: target I/O control block
104 * @iov: array of vectors that define I/O buffer
105 * @pos: offset in file to begin the operation
106 * @nr_segs: size of iovec array
107 *
108 * The presence of this routine in the address space ops vector means
109 * the NFS client supports direct I/O. However, we shunt off direct
110 * read and write requests before the VFS gets them, so this method
111 * should never be called.
Linus Torvalds1da177e2005-04-16 15:20:36 -0700112 */
Chuck Leverb8a32e22006-03-20 13:44:28 -0500113ssize_t nfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t pos, unsigned long nr_segs)
114{
Chuck Leverb8a32e22006-03-20 13:44:28 -0500115 dprintk("NFS: nfs_direct_IO (%s) off/no(%Ld/%lu) EINVAL\n",
Trond Myklebuste99170f2006-04-18 13:21:42 -0400116 iocb->ki_filp->f_dentry->d_name.name,
117 (long long) pos, nr_segs);
Chuck Leverb8a32e22006-03-20 13:44:28 -0500118
119 return -EINVAL;
120}
121
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400122static void nfs_free_user_pages(struct page **pages, int npages, int do_dirty)
Trond Myklebust6b45d852006-03-20 13:44:43 -0500123{
124 int i;
125 for (i = 0; i < npages; i++) {
126 struct page *page = pages[i];
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400127 if (do_dirty && !PageCompound(page))
Trond Myklebust6b45d852006-03-20 13:44:43 -0500128 set_page_dirty_lock(page);
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400129 page_cache_release(page);
Trond Myklebust6b45d852006-03-20 13:44:43 -0500130 }
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400131 kfree(pages);
Chuck Lever9c93ab72006-06-20 12:56:31 -0400132}
133
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400134static inline int nfs_get_user_pages(int rw, unsigned long user_addr, size_t size, struct page ***pages)
Chuck Lever9c93ab72006-06-20 12:56:31 -0400135{
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400136 int result = -ENOMEM;
137 unsigned long page_count;
138 size_t array_size;
139
140 page_count = (user_addr + size + PAGE_SIZE - 1) >> PAGE_SHIFT;
141 page_count -= user_addr >> PAGE_SHIFT;
142
143 array_size = (page_count * sizeof(struct page *));
144 *pages = kmalloc(array_size, GFP_KERNEL);
145 if (*pages) {
146 down_read(&current->mm->mmap_sem);
147 result = get_user_pages(current, current->mm, user_addr,
148 page_count, (rw == READ), 0,
149 *pages, NULL);
150 up_read(&current->mm->mmap_sem);
151 if (result != page_count) {
152 /*
153 * If we got fewer pages than expected from
154 * get_user_pages(), the user buffer runs off the
155 * end of a mapping; return EFAULT.
156 */
157 if (result >= 0) {
158 nfs_free_user_pages(*pages, result, 0);
159 result = -EFAULT;
160 } else
161 kfree(*pages);
162 *pages = NULL;
163 }
164 }
165 return result;
Trond Myklebust6b45d852006-03-20 13:44:43 -0500166}
167
Chuck Lever93619e52006-03-20 13:44:31 -0500168static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700169{
Linus Torvalds1da177e2005-04-16 15:20:36 -0700170 struct nfs_direct_req *dreq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700171
172 dreq = kmem_cache_alloc(nfs_direct_cachep, SLAB_KERNEL);
173 if (!dreq)
174 return NULL;
175
176 kref_init(&dreq->kref);
Trond Myklebustd72b7a62006-03-20 13:44:43 -0500177 init_completion(&dreq->completion);
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400178 INIT_LIST_HEAD(&dreq->list);
Trond Myklebustfad61492006-03-20 13:44:36 -0500179 INIT_LIST_HEAD(&dreq->rewrite_list);
Chuck Lever93619e52006-03-20 13:44:31 -0500180 dreq->iocb = NULL;
Trond Myklebusta8881f52006-03-20 13:44:36 -0500181 dreq->ctx = NULL;
Chuck Lever15ce4a02006-03-20 13:44:34 -0500182 spin_lock_init(&dreq->lock);
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400183 dreq->outstanding = 0;
Chuck Lever15ce4a02006-03-20 13:44:34 -0500184 dreq->count = 0;
185 dreq->error = 0;
Trond Myklebustfad61492006-03-20 13:44:36 -0500186 dreq->flags = 0;
Chuck Lever93619e52006-03-20 13:44:31 -0500187
188 return dreq;
189}
190
Linus Torvalds1da177e2005-04-16 15:20:36 -0700191static void nfs_direct_req_release(struct kref *kref)
192{
193 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
Trond Myklebusta8881f52006-03-20 13:44:36 -0500194
195 if (dreq->ctx != NULL)
196 put_nfs_open_context(dreq->ctx);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700197 kmem_cache_free(nfs_direct_cachep, dreq);
198}
199
Chuck Leverd4cc9482006-03-20 13:44:28 -0500200/*
Chuck Leverbc0fb202006-03-20 13:44:31 -0500201 * Collects and returns the final error value/byte-count.
202 */
203static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
204{
Chuck Lever15ce4a02006-03-20 13:44:34 -0500205 ssize_t result = -EIOCBQUEUED;
Chuck Leverbc0fb202006-03-20 13:44:31 -0500206
207 /* Async requests don't wait here */
208 if (dreq->iocb)
209 goto out;
210
Trond Myklebustd72b7a62006-03-20 13:44:43 -0500211 result = wait_for_completion_interruptible(&dreq->completion);
Chuck Leverbc0fb202006-03-20 13:44:31 -0500212
213 if (!result)
Chuck Lever15ce4a02006-03-20 13:44:34 -0500214 result = dreq->error;
Chuck Leverbc0fb202006-03-20 13:44:31 -0500215 if (!result)
Chuck Lever15ce4a02006-03-20 13:44:34 -0500216 result = dreq->count;
Chuck Leverbc0fb202006-03-20 13:44:31 -0500217
218out:
219 kref_put(&dreq->kref, nfs_direct_req_release);
220 return (ssize_t) result;
221}
222
223/*
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400224 * We must hold a reference to all the pages in this direct read request
225 * until the RPCs complete. This could be long *after* we are woken up in
226 * nfs_direct_wait (for instance, if someone hits ^C on a slow server).
227 *
228 * In addition, synchronous I/O uses a stack-allocated iocb. Thus we
229 * can't trust the iocb is still valid here if this is a synchronous
230 * request. If the waiter is woken prematurely, the iocb is long gone.
Chuck Lever63ab46a2006-03-20 13:44:31 -0500231 */
232static void nfs_direct_complete(struct nfs_direct_req *dreq)
233{
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400234 nfs_free_user_pages(dreq->pages, dreq->npages, 1);
235
Chuck Lever63ab46a2006-03-20 13:44:31 -0500236 if (dreq->iocb) {
Chuck Lever15ce4a02006-03-20 13:44:34 -0500237 long res = (long) dreq->error;
Chuck Lever63ab46a2006-03-20 13:44:31 -0500238 if (!res)
Chuck Lever15ce4a02006-03-20 13:44:34 -0500239 res = (long) dreq->count;
Chuck Lever63ab46a2006-03-20 13:44:31 -0500240 aio_complete(dreq->iocb, res, 0);
Trond Myklebustd72b7a62006-03-20 13:44:43 -0500241 }
242 complete_all(&dreq->completion);
Chuck Lever63ab46a2006-03-20 13:44:31 -0500243
244 kref_put(&dreq->kref, nfs_direct_req_release);
245}
246
247/*
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400248 * Note we also set the number of requests we have in the dreq when we are
249 * done. This prevents races with I/O completion so we will always wait
250 * until all requests have been dispatched and completed.
Chuck Lever06cf6f22006-06-20 12:56:49 -0400251 */
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400252static struct nfs_direct_req *nfs_direct_read_alloc(size_t nbytes, size_t rsize)
253{
254 struct list_head *list;
255 struct nfs_direct_req *dreq;
256 unsigned int rpages = (rsize + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
257
258 dreq = nfs_direct_req_alloc();
259 if (!dreq)
260 return NULL;
261
262 list = &dreq->list;
263 for(;;) {
264 struct nfs_read_data *data = nfs_readdata_alloc(rpages);
265
266 if (unlikely(!data)) {
267 while (!list_empty(list)) {
268 data = list_entry(list->next,
269 struct nfs_read_data, pages);
270 list_del(&data->pages);
271 nfs_readdata_free(data);
272 }
273 kref_put(&dreq->kref, nfs_direct_req_release);
274 return NULL;
275 }
276
277 INIT_LIST_HEAD(&data->pages);
278 list_add(&data->pages, list);
279
280 data->req = (struct nfs_page *) dreq;
281 dreq->outstanding++;
282 if (nbytes <= rsize)
283 break;
284 nbytes -= rsize;
285 }
286 kref_get(&dreq->kref);
287 return dreq;
288}
289
Trond Myklebustec06c092006-03-20 13:44:27 -0500290static void nfs_direct_read_result(struct rpc_task *task, void *calldata)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700291{
Trond Myklebustec06c092006-03-20 13:44:27 -0500292 struct nfs_read_data *data = calldata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700293 struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
294
Trond Myklebustec06c092006-03-20 13:44:27 -0500295 if (nfs_readpage_result(task, data) != 0)
296 return;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700297
Chuck Lever15ce4a02006-03-20 13:44:34 -0500298 spin_lock(&dreq->lock);
299
300 if (likely(task->tk_status >= 0))
301 dreq->count += data->res.count;
302 else
303 dreq->error = task->tk_status;
304
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400305 if (--dreq->outstanding) {
306 spin_unlock(&dreq->lock);
307 return;
308 }
Chuck Leverb1c59212006-06-20 12:55:19 -0400309
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400310 spin_unlock(&dreq->lock);
311 nfs_direct_complete(dreq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700312}
313
Trond Myklebustec06c092006-03-20 13:44:27 -0500314static const struct rpc_call_ops nfs_read_direct_ops = {
315 .rpc_call_done = nfs_direct_read_result,
316 .rpc_release = nfs_readdata_release,
317};
318
Chuck Leverd4cc9482006-03-20 13:44:28 -0500319/*
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400320 * For each nfs_read_data struct that was allocated on the list, dispatch
321 * an NFS READ operation
Linus Torvalds1da177e2005-04-16 15:20:36 -0700322 */
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400323static void nfs_direct_read_schedule(struct nfs_direct_req *dreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700324{
Trond Myklebusta8881f52006-03-20 13:44:36 -0500325 struct nfs_open_context *ctx = dreq->ctx;
326 struct inode *inode = ctx->dentry->d_inode;
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400327 struct list_head *list = &dreq->list;
328 struct page **pages = dreq->pages;
329 size_t count = dreq->user_count;
330 loff_t pos = dreq->pos;
Chuck Lever5dd602f2006-03-20 13:44:29 -0500331 size_t rsize = NFS_SERVER(inode)->rsize;
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400332 unsigned int curpage, pgbase;
Chuck Lever82b145c2006-06-20 12:57:03 -0400333
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400334 curpage = 0;
335 pgbase = dreq->user_addr & ~PAGE_MASK;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700336 do {
Chuck Lever82b145c2006-06-20 12:57:03 -0400337 struct nfs_read_data *data;
Chuck Lever5dd602f2006-03-20 13:44:29 -0500338 size_t bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700339
340 bytes = rsize;
341 if (count < rsize)
342 bytes = count;
343
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400344 BUG_ON(list_empty(list));
345 data = list_entry(list->next, struct nfs_read_data, pages);
346 list_del_init(&data->pages);
Chuck Lever06cf6f22006-06-20 12:56:49 -0400347
Linus Torvalds1da177e2005-04-16 15:20:36 -0700348 data->inode = inode;
349 data->cred = ctx->cred;
350 data->args.fh = NFS_FH(inode);
351 data->args.context = ctx;
Chuck Lever88467052006-03-20 13:44:34 -0500352 data->args.offset = pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700353 data->args.pgbase = pgbase;
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400354 data->args.pages = &pages[curpage];
Linus Torvalds1da177e2005-04-16 15:20:36 -0700355 data->args.count = bytes;
356 data->res.fattr = &data->fattr;
357 data->res.eof = 0;
358 data->res.count = bytes;
359
Trond Myklebustec06c092006-03-20 13:44:27 -0500360 rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
361 &nfs_read_direct_ops, data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700362 NFS_PROTO(inode)->read_setup(data);
363
364 data->task.tk_cookie = (unsigned long) inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700365
366 lock_kernel();
367 rpc_execute(&data->task);
368 unlock_kernel();
369
Chuck Lever606bbba2006-03-20 13:44:42 -0500370 dfprintk(VFS, "NFS: %5u initiated direct read call (req %s/%Ld, %zu bytes @ offset %Lu)\n",
Linus Torvalds1da177e2005-04-16 15:20:36 -0700371 data->task.tk_pid,
372 inode->i_sb->s_id,
373 (long long)NFS_FILEID(inode),
374 bytes,
375 (unsigned long long)data->args.offset);
376
Chuck Lever88467052006-03-20 13:44:34 -0500377 pos += bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700378 pgbase += bytes;
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400379 curpage += pgbase >> PAGE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700380 pgbase &= ~PAGE_MASK;
381
382 count -= bytes;
383 } while (count != 0);
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400384 BUG_ON(!list_empty(list));
Linus Torvalds1da177e2005-04-16 15:20:36 -0700385}
386
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400387static ssize_t nfs_direct_read(struct kiocb *iocb, unsigned long user_addr, size_t count, loff_t pos, struct page **pages, unsigned int nr_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700388{
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400389 ssize_t result;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700390 sigset_t oldset;
Chuck Lever99514f82006-03-20 13:44:30 -0500391 struct inode *inode = iocb->ki_filp->f_mapping->host;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700392 struct rpc_clnt *clnt = NFS_CLIENT(inode);
393 struct nfs_direct_req *dreq;
394
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400395 dreq = nfs_direct_read_alloc(count, NFS_SERVER(inode)->rsize);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700396 if (!dreq)
397 return -ENOMEM;
398
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400399 dreq->user_addr = user_addr;
400 dreq->user_count = count;
401 dreq->pos = pos;
402 dreq->pages = pages;
403 dreq->npages = nr_pages;
Chuck Lever91d5b472006-03-20 13:44:14 -0500404 dreq->inode = inode;
Trond Myklebusta8881f52006-03-20 13:44:36 -0500405 dreq->ctx = get_nfs_open_context((struct nfs_open_context *)iocb->ki_filp->private_data);
Chuck Lever487b8372006-03-20 13:44:30 -0500406 if (!is_sync_kiocb(iocb))
407 dreq->iocb = iocb;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700408
Chuck Lever91d5b472006-03-20 13:44:14 -0500409 nfs_add_stats(inode, NFSIOS_DIRECTREADBYTES, count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700410 rpc_clnt_sigmask(clnt, &oldset);
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400411 nfs_direct_read_schedule(dreq);
412 result = nfs_direct_wait(dreq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700413 rpc_clnt_sigunmask(clnt, &oldset);
414
415 return result;
416}
417
Trond Myklebustfad61492006-03-20 13:44:36 -0500418static void nfs_direct_free_writedata(struct nfs_direct_req *dreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700419{
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400420 list_splice_init(&dreq->rewrite_list, &dreq->list);
421 while (!list_empty(&dreq->list)) {
422 struct nfs_write_data *data = list_entry(dreq->list.next, struct nfs_write_data, pages);
Trond Myklebustfad61492006-03-20 13:44:36 -0500423 list_del(&data->pages);
424 nfs_writedata_release(data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700425 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700426}
427
Trond Myklebustfad61492006-03-20 13:44:36 -0500428#if defined(CONFIG_NFS_V3) || defined(CONFIG_NFS_V4)
429static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700430{
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400431 struct list_head *pos;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700432
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400433 list_splice_init(&dreq->rewrite_list, &dreq->list);
434 list_for_each(pos, &dreq->list)
435 dreq->outstanding++;
Trond Myklebustfad61492006-03-20 13:44:36 -0500436 dreq->count = 0;
437
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400438 nfs_direct_write_schedule(dreq, FLUSH_STABLE);
Trond Myklebustfad61492006-03-20 13:44:36 -0500439}
440
441static void nfs_direct_commit_result(struct rpc_task *task, void *calldata)
442{
443 struct nfs_write_data *data = calldata;
444 struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
445
446 /* Call the NFS version-specific code */
447 if (NFS_PROTO(data->inode)->commit_done(task, data) != 0)
448 return;
449 if (unlikely(task->tk_status < 0)) {
450 dreq->error = task->tk_status;
451 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
452 }
453 if (memcmp(&dreq->verf, &data->verf, sizeof(data->verf))) {
454 dprintk("NFS: %5u commit verify failed\n", task->tk_pid);
455 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
456 }
457
458 dprintk("NFS: %5u commit returned %d\n", task->tk_pid, task->tk_status);
459 nfs_direct_write_complete(dreq, data->inode);
460}
461
462static const struct rpc_call_ops nfs_commit_direct_ops = {
463 .rpc_call_done = nfs_direct_commit_result,
464 .rpc_release = nfs_commit_release,
465};
466
467static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
468{
Trond Myklebustfad61492006-03-20 13:44:36 -0500469 struct nfs_write_data *data = dreq->commit_data;
Trond Myklebustfad61492006-03-20 13:44:36 -0500470
471 data->inode = dreq->inode;
Trond Myklebusta8881f52006-03-20 13:44:36 -0500472 data->cred = dreq->ctx->cred;
Trond Myklebustfad61492006-03-20 13:44:36 -0500473
474 data->args.fh = NFS_FH(data->inode);
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400475 data->args.offset = dreq->pos;
476 data->args.count = dreq->user_count;
Trond Myklebustfad61492006-03-20 13:44:36 -0500477 data->res.count = 0;
478 data->res.fattr = &data->fattr;
479 data->res.verf = &data->verf;
480
481 rpc_init_task(&data->task, NFS_CLIENT(dreq->inode), RPC_TASK_ASYNC,
482 &nfs_commit_direct_ops, data);
483 NFS_PROTO(data->inode)->commit_setup(data, 0);
484
485 data->task.tk_priority = RPC_PRIORITY_NORMAL;
486 data->task.tk_cookie = (unsigned long)data->inode;
487 /* Note: task.tk_ops->rpc_release will free dreq->commit_data */
488 dreq->commit_data = NULL;
489
Trond Myklebuste99170f2006-04-18 13:21:42 -0400490 dprintk("NFS: %5u initiated commit call\n", data->task.tk_pid);
Trond Myklebustfad61492006-03-20 13:44:36 -0500491
492 lock_kernel();
493 rpc_execute(&data->task);
494 unlock_kernel();
495}
496
497static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
498{
499 int flags = dreq->flags;
500
501 dreq->flags = 0;
502 switch (flags) {
503 case NFS_ODIRECT_DO_COMMIT:
504 nfs_direct_commit_schedule(dreq);
505 break;
506 case NFS_ODIRECT_RESCHED_WRITES:
507 nfs_direct_write_reschedule(dreq);
508 break;
509 default:
510 nfs_end_data_update(inode);
511 if (dreq->commit_data != NULL)
512 nfs_commit_free(dreq->commit_data);
513 nfs_direct_free_writedata(dreq);
514 nfs_direct_complete(dreq);
515 }
516}
517
518static void nfs_alloc_commit_data(struct nfs_direct_req *dreq)
519{
520 dreq->commit_data = nfs_commit_alloc(0);
521 if (dreq->commit_data != NULL)
522 dreq->commit_data->req = (struct nfs_page *) dreq;
523}
524#else
525static inline void nfs_alloc_commit_data(struct nfs_direct_req *dreq)
526{
527 dreq->commit_data = NULL;
528}
529
530static void nfs_direct_write_complete(struct nfs_direct_req *dreq, struct inode *inode)
531{
532 nfs_end_data_update(inode);
533 nfs_direct_free_writedata(dreq);
534 nfs_direct_complete(dreq);
535}
536#endif
537
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400538static struct nfs_direct_req *nfs_direct_write_alloc(size_t nbytes, size_t wsize)
539{
540 struct list_head *list;
541 struct nfs_direct_req *dreq;
542 unsigned int wpages = (wsize + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
543
544 dreq = nfs_direct_req_alloc();
545 if (!dreq)
546 return NULL;
547
548 list = &dreq->list;
549 for(;;) {
550 struct nfs_write_data *data = nfs_writedata_alloc(wpages);
551
552 if (unlikely(!data)) {
553 while (!list_empty(list)) {
554 data = list_entry(list->next,
555 struct nfs_write_data, pages);
556 list_del(&data->pages);
557 nfs_writedata_free(data);
558 }
559 kref_put(&dreq->kref, nfs_direct_req_release);
560 return NULL;
561 }
562
563 INIT_LIST_HEAD(&data->pages);
564 list_add(&data->pages, list);
565
566 data->req = (struct nfs_page *) dreq;
567 dreq->outstanding++;
568 if (nbytes <= wsize)
569 break;
570 nbytes -= wsize;
571 }
572
573 nfs_alloc_commit_data(dreq);
574
575 kref_get(&dreq->kref);
576 return dreq;
577}
578
Chuck Lever462d5b32006-03-20 13:44:32 -0500579static void nfs_direct_write_result(struct rpc_task *task, void *calldata)
580{
581 struct nfs_write_data *data = calldata;
582 struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
583 int status = task->tk_status;
584
585 if (nfs_writeback_done(task, data) != 0)
586 return;
Chuck Lever462d5b32006-03-20 13:44:32 -0500587
Chuck Lever15ce4a02006-03-20 13:44:34 -0500588 spin_lock(&dreq->lock);
Chuck Lever462d5b32006-03-20 13:44:32 -0500589
Chuck Lever15ce4a02006-03-20 13:44:34 -0500590 if (likely(status >= 0))
591 dreq->count += data->res.count;
592 else
Trond Myklebustfad61492006-03-20 13:44:36 -0500593 dreq->error = task->tk_status;
Chuck Lever15ce4a02006-03-20 13:44:34 -0500594
Trond Myklebustfad61492006-03-20 13:44:36 -0500595 if (data->res.verf->committed != NFS_FILE_SYNC) {
596 switch (dreq->flags) {
597 case 0:
598 memcpy(&dreq->verf, &data->verf, sizeof(dreq->verf));
599 dreq->flags = NFS_ODIRECT_DO_COMMIT;
600 break;
601 case NFS_ODIRECT_DO_COMMIT:
602 if (memcmp(&dreq->verf, &data->verf, sizeof(dreq->verf))) {
603 dprintk("NFS: %5u write verify failed\n", task->tk_pid);
604 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
605 }
606 }
607 }
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400608 /* In case we have to resend */
609 data->args.stable = NFS_FILE_SYNC;
Trond Myklebustfad61492006-03-20 13:44:36 -0500610
611 spin_unlock(&dreq->lock);
612}
613
614/*
615 * NB: Return the value of the first error return code. Subsequent
616 * errors after the first one are ignored.
617 */
618static void nfs_direct_write_release(void *calldata)
619{
620 struct nfs_write_data *data = calldata;
621 struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
622
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400623 spin_lock(&dreq->lock);
624 if (--dreq->outstanding) {
625 spin_unlock(&dreq->lock);
626 return;
627 }
628 spin_unlock(&dreq->lock);
629
630 nfs_direct_write_complete(dreq, data->inode);
Chuck Lever462d5b32006-03-20 13:44:32 -0500631}
632
633static const struct rpc_call_ops nfs_write_direct_ops = {
634 .rpc_call_done = nfs_direct_write_result,
Trond Myklebustfad61492006-03-20 13:44:36 -0500635 .rpc_release = nfs_direct_write_release,
Chuck Lever462d5b32006-03-20 13:44:32 -0500636};
637
638/*
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400639 * For each nfs_write_data struct that was allocated on the list, dispatch
640 * an NFS WRITE operation
Chuck Lever462d5b32006-03-20 13:44:32 -0500641 */
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400642static void nfs_direct_write_schedule(struct nfs_direct_req *dreq, int sync)
Chuck Lever462d5b32006-03-20 13:44:32 -0500643{
Trond Myklebusta8881f52006-03-20 13:44:36 -0500644 struct nfs_open_context *ctx = dreq->ctx;
645 struct inode *inode = ctx->dentry->d_inode;
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400646 struct list_head *list = &dreq->list;
647 struct page **pages = dreq->pages;
648 size_t count = dreq->user_count;
649 loff_t pos = dreq->pos;
Chuck Lever462d5b32006-03-20 13:44:32 -0500650 size_t wsize = NFS_SERVER(inode)->wsize;
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400651 unsigned int curpage, pgbase;
Chuck Lever82b145c2006-06-20 12:57:03 -0400652
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400653 curpage = 0;
654 pgbase = dreq->user_addr & ~PAGE_MASK;
Chuck Lever462d5b32006-03-20 13:44:32 -0500655 do {
Chuck Lever82b145c2006-06-20 12:57:03 -0400656 struct nfs_write_data *data;
Chuck Lever462d5b32006-03-20 13:44:32 -0500657 size_t bytes;
658
659 bytes = wsize;
660 if (count < wsize)
661 bytes = count;
662
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400663 BUG_ON(list_empty(list));
664 data = list_entry(list->next, struct nfs_write_data, pages);
Trond Myklebustfad61492006-03-20 13:44:36 -0500665 list_move_tail(&data->pages, &dreq->rewrite_list);
Chuck Lever462d5b32006-03-20 13:44:32 -0500666
667 data->inode = inode;
668 data->cred = ctx->cred;
669 data->args.fh = NFS_FH(inode);
670 data->args.context = ctx;
Chuck Lever88467052006-03-20 13:44:34 -0500671 data->args.offset = pos;
Chuck Lever462d5b32006-03-20 13:44:32 -0500672 data->args.pgbase = pgbase;
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400673 data->args.pages = &pages[curpage];
Chuck Lever462d5b32006-03-20 13:44:32 -0500674 data->args.count = bytes;
675 data->res.fattr = &data->fattr;
676 data->res.count = bytes;
Chuck Lever47989d72006-03-20 13:44:32 -0500677 data->res.verf = &data->verf;
Chuck Lever462d5b32006-03-20 13:44:32 -0500678
679 rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
680 &nfs_write_direct_ops, data);
Trond Myklebustfad61492006-03-20 13:44:36 -0500681 NFS_PROTO(inode)->write_setup(data, sync);
Chuck Lever462d5b32006-03-20 13:44:32 -0500682
683 data->task.tk_priority = RPC_PRIORITY_NORMAL;
684 data->task.tk_cookie = (unsigned long) inode;
685
686 lock_kernel();
687 rpc_execute(&data->task);
688 unlock_kernel();
689
Chuck Lever606bbba2006-03-20 13:44:42 -0500690 dfprintk(VFS, "NFS: %5u initiated direct write call (req %s/%Ld, %zu bytes @ offset %Lu)\n",
Chuck Lever462d5b32006-03-20 13:44:32 -0500691 data->task.tk_pid,
692 inode->i_sb->s_id,
693 (long long)NFS_FILEID(inode),
694 bytes,
695 (unsigned long long)data->args.offset);
696
Chuck Lever88467052006-03-20 13:44:34 -0500697 pos += bytes;
Chuck Lever462d5b32006-03-20 13:44:32 -0500698 pgbase += bytes;
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400699 curpage += pgbase >> PAGE_SHIFT;
Chuck Lever462d5b32006-03-20 13:44:32 -0500700 pgbase &= ~PAGE_MASK;
701
702 count -= bytes;
703 } while (count != 0);
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400704 BUG_ON(!list_empty(list));
Chuck Lever462d5b32006-03-20 13:44:32 -0500705}
706
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400707static ssize_t nfs_direct_write(struct kiocb *iocb, unsigned long user_addr, size_t count, loff_t pos, struct page **pages, int nr_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700708{
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400709 ssize_t result;
Chuck Lever462d5b32006-03-20 13:44:32 -0500710 sigset_t oldset;
Chuck Leverc89f2ee2006-03-20 13:44:33 -0500711 struct inode *inode = iocb->ki_filp->f_mapping->host;
Chuck Lever462d5b32006-03-20 13:44:32 -0500712 struct rpc_clnt *clnt = NFS_CLIENT(inode);
713 struct nfs_direct_req *dreq;
Trond Myklebustfad61492006-03-20 13:44:36 -0500714 size_t wsize = NFS_SERVER(inode)->wsize;
715 int sync = 0;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700716
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400717 dreq = nfs_direct_write_alloc(count, wsize);
Chuck Lever462d5b32006-03-20 13:44:32 -0500718 if (!dreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719 return -ENOMEM;
Trond Myklebustfad61492006-03-20 13:44:36 -0500720 if (dreq->commit_data == NULL || count < wsize)
721 sync = FLUSH_STABLE;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700722
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400723 dreq->user_addr = user_addr;
724 dreq->user_count = count;
725 dreq->pos = pos;
726 dreq->pages = pages;
727 dreq->npages = nr_pages;
Chuck Leverc89f2ee2006-03-20 13:44:33 -0500728 dreq->inode = inode;
Trond Myklebusta8881f52006-03-20 13:44:36 -0500729 dreq->ctx = get_nfs_open_context((struct nfs_open_context *)iocb->ki_filp->private_data);
Chuck Leverc89f2ee2006-03-20 13:44:33 -0500730 if (!is_sync_kiocb(iocb))
731 dreq->iocb = iocb;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700732
Chuck Lever47989d72006-03-20 13:44:32 -0500733 nfs_add_stats(inode, NFSIOS_DIRECTWRITTENBYTES, count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700734
735 nfs_begin_data_update(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700736
Chuck Lever462d5b32006-03-20 13:44:32 -0500737 rpc_clnt_sigmask(clnt, &oldset);
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400738 nfs_direct_write_schedule(dreq, sync);
739 result = nfs_direct_wait(dreq);
Chuck Lever462d5b32006-03-20 13:44:32 -0500740 rpc_clnt_sigunmask(clnt, &oldset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741
Linus Torvalds1da177e2005-04-16 15:20:36 -0700742 return result;
743}
744
745/**
746 * nfs_file_direct_read - file direct read operation for NFS files
747 * @iocb: target I/O control block
748 * @buf: user's buffer into which to read data
Chuck Lever88467052006-03-20 13:44:34 -0500749 * @count: number of bytes to read
750 * @pos: byte offset in file where reading starts
Linus Torvalds1da177e2005-04-16 15:20:36 -0700751 *
752 * We use this function for direct reads instead of calling
753 * generic_file_aio_read() in order to avoid gfar's check to see if
754 * the request starts before the end of the file. For that check
755 * to work, we must generate a GETATTR before each direct read, and
756 * even then there is a window between the GETATTR and the subsequent
Chuck Lever88467052006-03-20 13:44:34 -0500757 * READ where the file size could change. Our preference is simply
Linus Torvalds1da177e2005-04-16 15:20:36 -0700758 * to do all reads the application wants, and the server will take
759 * care of managing the end of file boundary.
Chuck Lever88467052006-03-20 13:44:34 -0500760 *
Linus Torvalds1da177e2005-04-16 15:20:36 -0700761 * This function also eliminates unnecessarily updating the file's
762 * atime locally, as the NFS server sets the file's atime, and this
763 * client must read the updated atime from the server back into its
764 * cache.
765 */
Chuck Leverd4cc9482006-03-20 13:44:28 -0500766ssize_t nfs_file_direct_read(struct kiocb *iocb, char __user *buf, size_t count, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700767{
768 ssize_t retval = -EINVAL;
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400769 int page_count;
770 struct page **pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700771 struct file *file = iocb->ki_filp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700772 struct address_space *mapping = file->f_mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700773
Chuck Leverce1a8e62005-11-30 18:08:17 -0500774 dprintk("nfs: direct read(%s/%s, %lu@%Ld)\n",
Chuck Lever0bbacc42005-11-01 16:53:32 -0500775 file->f_dentry->d_parent->d_name.name,
776 file->f_dentry->d_name.name,
Chuck Leverce1a8e62005-11-30 18:08:17 -0500777 (unsigned long) count, (long long) pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700778
Linus Torvalds1da177e2005-04-16 15:20:36 -0700779 if (count < 0)
780 goto out;
781 retval = -EFAULT;
Chuck Lever0cdd80d2006-03-20 13:44:29 -0500782 if (!access_ok(VERIFY_WRITE, buf, count))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700783 goto out;
784 retval = 0;
785 if (!count)
786 goto out;
787
Trond Myklebust29884df2005-12-13 16:13:54 -0500788 retval = nfs_sync_mapping(mapping);
789 if (retval)
790 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700791
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400792 retval = nfs_get_user_pages(READ, (unsigned long) buf,
793 count, &pages);
794 if (retval < 0)
795 goto out;
796 page_count = retval;
797
798 retval = nfs_direct_read(iocb, (unsigned long) buf, count, pos,
799 pages, page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700800 if (retval > 0)
Chuck Lever0cdd80d2006-03-20 13:44:29 -0500801 iocb->ki_pos = pos + retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700802
803out:
804 return retval;
805}
806
807/**
808 * nfs_file_direct_write - file direct write operation for NFS files
809 * @iocb: target I/O control block
810 * @buf: user's buffer from which to write data
Chuck Lever88467052006-03-20 13:44:34 -0500811 * @count: number of bytes to write
812 * @pos: byte offset in file where writing starts
Linus Torvalds1da177e2005-04-16 15:20:36 -0700813 *
814 * We use this function for direct writes instead of calling
815 * generic_file_aio_write() in order to avoid taking the inode
816 * semaphore and updating the i_size. The NFS server will set
817 * the new i_size and this client must read the updated size
818 * back into its cache. We let the server do generic write
819 * parameter checking and report problems.
820 *
821 * We also avoid an unnecessary invocation of generic_osync_inode(),
822 * as it is fairly meaningless to sync the metadata of an NFS file.
823 *
824 * We eliminate local atime updates, see direct read above.
825 *
826 * We avoid unnecessary page cache invalidations for normal cached
827 * readers of this file.
828 *
829 * Note that O_APPEND is not supported for NFS direct writes, as there
830 * is no atomic O_APPEND write facility in the NFS protocol.
831 */
Chuck Leverd4cc9482006-03-20 13:44:28 -0500832ssize_t nfs_file_direct_write(struct kiocb *iocb, const char __user *buf, size_t count, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700833{
Chuck Leverce1a8e62005-11-30 18:08:17 -0500834 ssize_t retval;
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400835 int page_count;
836 struct page **pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700837 struct file *file = iocb->ki_filp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700838 struct address_space *mapping = file->f_mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700839
Chuck Leverce1a8e62005-11-30 18:08:17 -0500840 dfprintk(VFS, "nfs: direct write(%s/%s, %lu@%Ld)\n",
Chuck Lever0bbacc42005-11-01 16:53:32 -0500841 file->f_dentry->d_parent->d_name.name,
Chuck Leverce1a8e62005-11-30 18:08:17 -0500842 file->f_dentry->d_name.name,
843 (unsigned long) count, (long long) pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700844
Chuck Leverce1a8e62005-11-30 18:08:17 -0500845 retval = generic_write_checks(file, &pos, &count, 0);
846 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700847 goto out;
Chuck Leverce1a8e62005-11-30 18:08:17 -0500848
849 retval = -EINVAL;
850 if ((ssize_t) count < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700851 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700852 retval = 0;
853 if (!count)
854 goto out;
Chuck Leverce1a8e62005-11-30 18:08:17 -0500855
856 retval = -EFAULT;
Chuck Lever47989d72006-03-20 13:44:32 -0500857 if (!access_ok(VERIFY_READ, buf, count))
Chuck Leverce1a8e62005-11-30 18:08:17 -0500858 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700859
Trond Myklebust29884df2005-12-13 16:13:54 -0500860 retval = nfs_sync_mapping(mapping);
861 if (retval)
862 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700863
Trond Myklebustccf01ef2006-06-25 06:27:31 -0400864 retval = nfs_get_user_pages(WRITE, (unsigned long) buf,
865 count, &pages);
866 if (retval < 0)
867 goto out;
868 page_count = retval;
869
870 retval = nfs_direct_write(iocb, (unsigned long) buf, count,
871 pos, pages, page_count);
Chuck Lever9eafa8c2006-03-20 13:44:33 -0500872
873 /*
874 * XXX: nfs_end_data_update() already ensures this file's
875 * cached data is subsequently invalidated. Do we really
876 * need to call invalidate_inode_pages2() again here?
877 *
878 * For aio writes, this invalidation will almost certainly
879 * occur before the writes complete. Kind of racey.
880 */
Linus Torvalds1da177e2005-04-16 15:20:36 -0700881 if (mapping->nrpages)
882 invalidate_inode_pages2(mapping);
Chuck Lever9eafa8c2006-03-20 13:44:33 -0500883
Linus Torvalds1da177e2005-04-16 15:20:36 -0700884 if (retval > 0)
Chuck Leverce1a8e62005-11-30 18:08:17 -0500885 iocb->ki_pos = pos + retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700886
887out:
888 return retval;
889}
890
Chuck Lever88467052006-03-20 13:44:34 -0500891/**
892 * nfs_init_directcache - create a slab cache for nfs_direct_req structures
893 *
894 */
David Howellsf7b422b2006-06-09 09:34:33 -0400895int __init nfs_init_directcache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700896{
897 nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
898 sizeof(struct nfs_direct_req),
Paul Jacksonfffb60f2006-03-24 03:16:06 -0800899 0, (SLAB_RECLAIM_ACCOUNT|
900 SLAB_MEM_SPREAD),
Linus Torvalds1da177e2005-04-16 15:20:36 -0700901 NULL, NULL);
902 if (nfs_direct_cachep == NULL)
903 return -ENOMEM;
904
905 return 0;
906}
907
Chuck Lever88467052006-03-20 13:44:34 -0500908/**
David Howellsf7b422b2006-06-09 09:34:33 -0400909 * nfs_destroy_directcache - destroy the slab cache for nfs_direct_req structures
Chuck Lever88467052006-03-20 13:44:34 -0500910 *
911 */
David Brownell266bee82006-06-27 12:59:15 -0700912void nfs_destroy_directcache(void)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700913{
914 if (kmem_cache_destroy(nfs_direct_cachep))
915 printk(KERN_INFO "nfs_direct_cache: not all structures were freed\n");
916}