blob: dea3239cdded28df693c7d90b0c682ca98425bc3 [file] [log] [blame]
Linus Torvalds1da177e2005-04-16 15:20:36 -07001/*
2 * linux/fs/nfs/direct.c
3 *
4 * Copyright (C) 2003 by Chuck Lever <cel@netapp.com>
5 *
6 * High-performance uncached I/O for the Linux NFS client
7 *
8 * There are important applications whose performance or correctness
9 * depends on uncached access to file data. Database clusters
10 * (multiple copies of the same instance running on separate hosts)
11 * implement their own cache coherency protocol that subsumes file
12 * system cache protocols. Applications that process datasets
13 * considerably larger than the client's memory do not always benefit
14 * from a local cache. A streaming video server, for instance, has no
15 * need to cache the contents of a file.
16 *
17 * When an application requests uncached I/O, all read and write requests
18 * are made directly to the server; data stored or fetched via these
19 * requests is not cached in the Linux page cache. The client does not
20 * correct unaligned requests from applications. All requested bytes are
21 * held on permanent storage before a direct write system call returns to
22 * an application.
23 *
24 * Solaris implements an uncached I/O facility called directio() that
25 * is used for backups and sequential I/O to very large files. Solaris
26 * also supports uncaching whole NFS partitions with "-o forcedirectio,"
27 * an undocumented mount option.
28 *
29 * Designed by Jeff Kimmel, Chuck Lever, and Trond Myklebust, with
30 * help from Andrew Morton.
31 *
32 * 18 Dec 2001 Initial implementation for 2.4 --cel
33 * 08 Jul 2002 Version for 2.4.19, with bug fixes --trondmy
34 * 08 Jun 2003 Port to 2.5 APIs --cel
35 * 31 Mar 2004 Handle direct I/O without VFS support --cel
36 * 15 Sep 2004 Parallel async reads --cel
37 *
38 */
39
40#include <linux/config.h>
41#include <linux/errno.h>
42#include <linux/sched.h>
43#include <linux/kernel.h>
44#include <linux/smp_lock.h>
45#include <linux/file.h>
46#include <linux/pagemap.h>
47#include <linux/kref.h>
48
49#include <linux/nfs_fs.h>
50#include <linux/nfs_page.h>
51#include <linux/sunrpc/clnt.h>
52
53#include <asm/system.h>
54#include <asm/uaccess.h>
55#include <asm/atomic.h>
56
Chuck Lever91d5b472006-03-20 13:44:14 -050057#include "iostat.h"
58
Linus Torvalds1da177e2005-04-16 15:20:36 -070059#define NFSDBG_FACILITY NFSDBG_VFS
60#define MAX_DIRECTIO_SIZE (4096UL << PAGE_SHIFT)
61
Trond Myklebust143f4122006-03-13 21:20:46 -080062static void nfs_free_user_pages(struct page **pages, int npages, int do_dirty);
Linus Torvalds1da177e2005-04-16 15:20:36 -070063static kmem_cache_t *nfs_direct_cachep;
64
65/*
66 * This represents a set of asynchronous requests that we're waiting on
67 */
68struct nfs_direct_req {
69 struct kref kref; /* release manager */
70 struct list_head list; /* nfs_read_data structs */
Chuck Lever99514f82006-03-20 13:44:30 -050071 struct file * filp; /* file descriptor */
72 struct kiocb * iocb; /* controlling i/o request */
Linus Torvalds1da177e2005-04-16 15:20:36 -070073 wait_queue_head_t wait; /* wait for i/o completion */
Chuck Lever91d5b472006-03-20 13:44:14 -050074 struct inode * inode; /* target file of I/O */
Linus Torvalds1da177e2005-04-16 15:20:36 -070075 struct page ** pages; /* pages in our buffer */
76 unsigned int npages; /* count of pages */
77 atomic_t complete, /* i/os we're waiting for */
78 count, /* bytes actually processed */
79 error; /* any reported error */
80};
81
82
83/**
Chuck Leverb8a32e22006-03-20 13:44:28 -050084 * nfs_direct_IO - NFS address space operation for direct I/O
85 * @rw: direction (read or write)
86 * @iocb: target I/O control block
87 * @iov: array of vectors that define I/O buffer
88 * @pos: offset in file to begin the operation
89 * @nr_segs: size of iovec array
90 *
91 * The presence of this routine in the address space ops vector means
92 * the NFS client supports direct I/O. However, we shunt off direct
93 * read and write requests before the VFS gets them, so this method
94 * should never be called.
95 */
96ssize_t nfs_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov, loff_t pos, unsigned long nr_segs)
97{
98 struct dentry *dentry = iocb->ki_filp->f_dentry;
99
100 dprintk("NFS: nfs_direct_IO (%s) off/no(%Ld/%lu) EINVAL\n",
101 dentry->d_name.name, (long long) pos, nr_segs);
102
103 return -EINVAL;
104}
105
Chuck Leverd4cc9482006-03-20 13:44:28 -0500106static inline int nfs_get_user_pages(int rw, unsigned long user_addr, size_t size, struct page ***pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700107{
108 int result = -ENOMEM;
109 unsigned long page_count;
110 size_t array_size;
111
112 /* set an arbitrary limit to prevent type overflow */
113 /* XXX: this can probably be as large as INT_MAX */
114 if (size > MAX_DIRECTIO_SIZE) {
115 *pages = NULL;
116 return -EFBIG;
117 }
118
119 page_count = (user_addr + size + PAGE_SIZE - 1) >> PAGE_SHIFT;
120 page_count -= user_addr >> PAGE_SHIFT;
121
122 array_size = (page_count * sizeof(struct page *));
123 *pages = kmalloc(array_size, GFP_KERNEL);
124 if (*pages) {
125 down_read(&current->mm->mmap_sem);
126 result = get_user_pages(current, current->mm, user_addr,
127 page_count, (rw == READ), 0,
128 *pages, NULL);
129 up_read(&current->mm->mmap_sem);
Trond Myklebust143f4122006-03-13 21:20:46 -0800130 /*
131 * If we got fewer pages than expected from get_user_pages(),
132 * the user buffer runs off the end of a mapping; return EFAULT.
133 */
134 if (result >= 0 && result < page_count) {
135 nfs_free_user_pages(*pages, result, 0);
136 *pages = NULL;
137 result = -EFAULT;
138 }
Linus Torvalds1da177e2005-04-16 15:20:36 -0700139 }
140 return result;
141}
142
Chuck Leverd4cc9482006-03-20 13:44:28 -0500143static void nfs_free_user_pages(struct page **pages, int npages, int do_dirty)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700144{
145 int i;
146 for (i = 0; i < npages; i++) {
Trond Myklebust566dd602006-01-03 09:55:35 +0100147 struct page *page = pages[i];
148 if (do_dirty && !PageCompound(page))
149 set_page_dirty_lock(page);
150 page_cache_release(page);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700151 }
152 kfree(pages);
153}
154
Chuck Lever93619e52006-03-20 13:44:31 -0500155static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
156{
157 struct nfs_direct_req *dreq;
158
159 dreq = kmem_cache_alloc(nfs_direct_cachep, SLAB_KERNEL);
160 if (!dreq)
161 return NULL;
162
163 kref_init(&dreq->kref);
164 init_waitqueue_head(&dreq->wait);
165 INIT_LIST_HEAD(&dreq->list);
166 dreq->iocb = NULL;
167 atomic_set(&dreq->count, 0);
168 atomic_set(&dreq->error, 0);
169
170 return dreq;
171}
172
Linus Torvalds1da177e2005-04-16 15:20:36 -0700173static void nfs_direct_req_release(struct kref *kref)
174{
175 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
176 kmem_cache_free(nfs_direct_cachep, dreq);
177}
178
Chuck Leverd4cc9482006-03-20 13:44:28 -0500179/*
Chuck Leverbc0fb202006-03-20 13:44:31 -0500180 * Collects and returns the final error value/byte-count.
181 */
182static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
183{
184 int result = -EIOCBQUEUED;
185
186 /* Async requests don't wait here */
187 if (dreq->iocb)
188 goto out;
189
190 result = wait_event_interruptible(dreq->wait,
191 (atomic_read(&dreq->complete) == 0));
192
193 if (!result)
194 result = atomic_read(&dreq->error);
195 if (!result)
196 result = atomic_read(&dreq->count);
197
198out:
199 kref_put(&dreq->kref, nfs_direct_req_release);
200 return (ssize_t) result;
201}
202
203/*
Chuck Lever63ab46a2006-03-20 13:44:31 -0500204 * We must hold a reference to all the pages in this direct read request
205 * until the RPCs complete. This could be long *after* we are woken up in
206 * nfs_direct_wait (for instance, if someone hits ^C on a slow server).
207 *
208 * In addition, synchronous I/O uses a stack-allocated iocb. Thus we
209 * can't trust the iocb is still valid here if this is a synchronous
210 * request. If the waiter is woken prematurely, the iocb is long gone.
211 */
212static void nfs_direct_complete(struct nfs_direct_req *dreq)
213{
214 nfs_free_user_pages(dreq->pages, dreq->npages, 1);
215
216 if (dreq->iocb) {
217 long res = atomic_read(&dreq->error);
218 if (!res)
219 res = atomic_read(&dreq->count);
220 aio_complete(dreq->iocb, res, 0);
221 } else
222 wake_up(&dreq->wait);
223
224 kref_put(&dreq->kref, nfs_direct_req_release);
225}
226
227/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700228 * Note we also set the number of requests we have in the dreq when we are
229 * done. This prevents races with I/O completion so we will always wait
230 * until all requests have been dispatched and completed.
231 */
Chuck Lever5dd602f2006-03-20 13:44:29 -0500232static struct nfs_direct_req *nfs_direct_read_alloc(size_t nbytes, size_t rsize)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700233{
234 struct list_head *list;
235 struct nfs_direct_req *dreq;
236 unsigned int reads = 0;
Chuck Lever40859d72005-11-30 18:09:02 -0500237 unsigned int rpages = (rsize + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700238
Chuck Lever93619e52006-03-20 13:44:31 -0500239 dreq = nfs_direct_req_alloc();
Linus Torvalds1da177e2005-04-16 15:20:36 -0700240 if (!dreq)
241 return NULL;
242
Linus Torvalds1da177e2005-04-16 15:20:36 -0700243 list = &dreq->list;
244 for(;;) {
Chuck Lever40859d72005-11-30 18:09:02 -0500245 struct nfs_read_data *data = nfs_readdata_alloc(rpages);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700246
247 if (unlikely(!data)) {
248 while (!list_empty(list)) {
249 data = list_entry(list->next,
250 struct nfs_read_data, pages);
251 list_del(&data->pages);
252 nfs_readdata_free(data);
253 }
254 kref_put(&dreq->kref, nfs_direct_req_release);
255 return NULL;
256 }
257
258 INIT_LIST_HEAD(&data->pages);
259 list_add(&data->pages, list);
260
261 data->req = (struct nfs_page *) dreq;
262 reads++;
263 if (nbytes <= rsize)
264 break;
265 nbytes -= rsize;
266 }
267 kref_get(&dreq->kref);
268 atomic_set(&dreq->complete, reads);
269 return dreq;
270}
271
Trond Myklebustec06c092006-03-20 13:44:27 -0500272static void nfs_direct_read_result(struct rpc_task *task, void *calldata)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700273{
Trond Myklebustec06c092006-03-20 13:44:27 -0500274 struct nfs_read_data *data = calldata;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700275 struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
276
Trond Myklebustec06c092006-03-20 13:44:27 -0500277 if (nfs_readpage_result(task, data) != 0)
278 return;
279 if (likely(task->tk_status >= 0))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700280 atomic_add(data->res.count, &dreq->count);
281 else
Trond Myklebustec06c092006-03-20 13:44:27 -0500282 atomic_set(&dreq->error, task->tk_status);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700283
Chuck Lever63ab46a2006-03-20 13:44:31 -0500284 if (unlikely(atomic_dec_and_test(&dreq->complete)))
285 nfs_direct_complete(dreq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700286}
287
Trond Myklebustec06c092006-03-20 13:44:27 -0500288static const struct rpc_call_ops nfs_read_direct_ops = {
289 .rpc_call_done = nfs_direct_read_result,
290 .rpc_release = nfs_readdata_release,
291};
292
Chuck Leverd4cc9482006-03-20 13:44:28 -0500293/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700294 * For each nfs_read_data struct that was allocated on the list, dispatch
295 * an NFS READ operation
296 */
Chuck Lever99514f82006-03-20 13:44:30 -0500297static void nfs_direct_read_schedule(struct nfs_direct_req *dreq, unsigned long user_addr, size_t count, loff_t file_offset)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700298{
Chuck Lever99514f82006-03-20 13:44:30 -0500299 struct file *file = dreq->filp;
300 struct inode *inode = file->f_mapping->host;
301 struct nfs_open_context *ctx = (struct nfs_open_context *)
302 file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700303 struct list_head *list = &dreq->list;
304 struct page **pages = dreq->pages;
Chuck Lever5dd602f2006-03-20 13:44:29 -0500305 size_t rsize = NFS_SERVER(inode)->rsize;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700306 unsigned int curpage, pgbase;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700307
308 curpage = 0;
309 pgbase = user_addr & ~PAGE_MASK;
310 do {
311 struct nfs_read_data *data;
Chuck Lever5dd602f2006-03-20 13:44:29 -0500312 size_t bytes;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700313
314 bytes = rsize;
315 if (count < rsize)
316 bytes = count;
317
318 data = list_entry(list->next, struct nfs_read_data, pages);
319 list_del_init(&data->pages);
320
321 data->inode = inode;
322 data->cred = ctx->cred;
323 data->args.fh = NFS_FH(inode);
324 data->args.context = ctx;
325 data->args.offset = file_offset;
326 data->args.pgbase = pgbase;
327 data->args.pages = &pages[curpage];
328 data->args.count = bytes;
329 data->res.fattr = &data->fattr;
330 data->res.eof = 0;
331 data->res.count = bytes;
332
Trond Myklebustec06c092006-03-20 13:44:27 -0500333 rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
334 &nfs_read_direct_ops, data);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700335 NFS_PROTO(inode)->read_setup(data);
336
337 data->task.tk_cookie = (unsigned long) inode;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700338
339 lock_kernel();
340 rpc_execute(&data->task);
341 unlock_kernel();
342
343 dfprintk(VFS, "NFS: %4d initiated direct read call (req %s/%Ld, %u bytes @ offset %Lu)\n",
344 data->task.tk_pid,
345 inode->i_sb->s_id,
346 (long long)NFS_FILEID(inode),
347 bytes,
348 (unsigned long long)data->args.offset);
349
350 file_offset += bytes;
351 pgbase += bytes;
352 curpage += pgbase >> PAGE_SHIFT;
353 pgbase &= ~PAGE_MASK;
354
355 count -= bytes;
356 } while (count != 0);
357}
358
Chuck Lever99514f82006-03-20 13:44:30 -0500359static ssize_t nfs_direct_read(struct kiocb *iocb, unsigned long user_addr, size_t count, loff_t file_offset, struct page **pages, unsigned int nr_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700360{
361 ssize_t result;
362 sigset_t oldset;
Chuck Lever99514f82006-03-20 13:44:30 -0500363 struct inode *inode = iocb->ki_filp->f_mapping->host;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700364 struct rpc_clnt *clnt = NFS_CLIENT(inode);
365 struct nfs_direct_req *dreq;
366
367 dreq = nfs_direct_read_alloc(count, NFS_SERVER(inode)->rsize);
368 if (!dreq)
369 return -ENOMEM;
370
371 dreq->pages = pages;
372 dreq->npages = nr_pages;
Chuck Lever91d5b472006-03-20 13:44:14 -0500373 dreq->inode = inode;
Chuck Lever99514f82006-03-20 13:44:30 -0500374 dreq->filp = iocb->ki_filp;
Chuck Lever487b8372006-03-20 13:44:30 -0500375 if (!is_sync_kiocb(iocb))
376 dreq->iocb = iocb;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700377
Chuck Lever91d5b472006-03-20 13:44:14 -0500378 nfs_add_stats(inode, NFSIOS_DIRECTREADBYTES, count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700379 rpc_clnt_sigmask(clnt, &oldset);
Chuck Lever99514f82006-03-20 13:44:30 -0500380 nfs_direct_read_schedule(dreq, user_addr, count, file_offset);
Chuck Leverbc0fb202006-03-20 13:44:31 -0500381 result = nfs_direct_wait(dreq);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700382 rpc_clnt_sigunmask(clnt, &oldset);
383
384 return result;
385}
386
Chuck Lever462d5b32006-03-20 13:44:32 -0500387static struct nfs_direct_req *nfs_direct_write_alloc(size_t nbytes, size_t wsize)
388{
389 struct list_head *list;
390 struct nfs_direct_req *dreq;
391 unsigned int writes = 0;
392 unsigned int wpages = (wsize + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
393
394 dreq = nfs_direct_req_alloc();
395 if (!dreq)
396 return NULL;
397
398 list = &dreq->list;
399 for(;;) {
400 struct nfs_write_data *data = nfs_writedata_alloc(wpages);
401
402 if (unlikely(!data)) {
403 while (!list_empty(list)) {
404 data = list_entry(list->next,
405 struct nfs_write_data, pages);
406 list_del(&data->pages);
407 nfs_writedata_free(data);
408 }
409 kref_put(&dreq->kref, nfs_direct_req_release);
410 return NULL;
411 }
412
413 INIT_LIST_HEAD(&data->pages);
414 list_add(&data->pages, list);
415
416 data->req = (struct nfs_page *) dreq;
417 writes++;
418 if (nbytes <= wsize)
419 break;
420 nbytes -= wsize;
421 }
422 kref_get(&dreq->kref);
423 atomic_set(&dreq->complete, writes);
424 return dreq;
425}
426
427/*
428 * Collects and returns the final error value/byte-count.
429 */
430static ssize_t nfs_direct_write_wait(struct nfs_direct_req *dreq, int intr)
431{
432 int result = 0;
433
434 if (intr) {
435 result = wait_event_interruptible(dreq->wait,
436 (atomic_read(&dreq->complete) == 0));
437 } else {
438 wait_event(dreq->wait, (atomic_read(&dreq->complete) == 0));
439 }
440
441 if (!result)
442 result = atomic_read(&dreq->error);
443 if (!result)
444 result = atomic_read(&dreq->count);
445
446 kref_put(&dreq->kref, nfs_direct_req_release);
447 return (ssize_t) result;
448}
449
450static void nfs_direct_write_result(struct rpc_task *task, void *calldata)
451{
452 struct nfs_write_data *data = calldata;
453 struct nfs_direct_req *dreq = (struct nfs_direct_req *) data->req;
454 int status = task->tk_status;
455
456 if (nfs_writeback_done(task, data) != 0)
457 return;
458 /* If the server fell back to an UNSTABLE write, it's an error. */
459 if (unlikely(data->res.verf->committed != NFS_FILE_SYNC))
460 status = -EIO;
461
462 if (likely(status >= 0))
463 atomic_add(data->res.count, &dreq->count);
464 else
465 atomic_set(&dreq->error, status);
466
467 if (unlikely(atomic_dec_and_test(&dreq->complete)))
468 nfs_direct_complete(dreq);
469}
470
471static const struct rpc_call_ops nfs_write_direct_ops = {
472 .rpc_call_done = nfs_direct_write_result,
473 .rpc_release = nfs_writedata_release,
474};
475
476/*
477 * For each nfs_write_data struct that was allocated on the list, dispatch
478 * an NFS WRITE operation
479 *
480 * XXX: For now, support only FILE_SYNC writes. Later we may add
481 * support for UNSTABLE + COMMIT.
482 */
483static void nfs_direct_write_schedule(struct nfs_direct_req *dreq, struct inode *inode, struct nfs_open_context *ctx, unsigned long user_addr, size_t count, loff_t file_offset)
484{
485 struct list_head *list = &dreq->list;
486 struct page **pages = dreq->pages;
487 size_t wsize = NFS_SERVER(inode)->wsize;
488 unsigned int curpage, pgbase;
489
490 curpage = 0;
491 pgbase = user_addr & ~PAGE_MASK;
492 do {
493 struct nfs_write_data *data;
494 size_t bytes;
495
496 bytes = wsize;
497 if (count < wsize)
498 bytes = count;
499
500 data = list_entry(list->next, struct nfs_write_data, pages);
501 list_del_init(&data->pages);
502
503 data->inode = inode;
504 data->cred = ctx->cred;
505 data->args.fh = NFS_FH(inode);
506 data->args.context = ctx;
507 data->args.offset = file_offset;
508 data->args.pgbase = pgbase;
509 data->args.pages = &pages[curpage];
510 data->args.count = bytes;
511 data->res.fattr = &data->fattr;
512 data->res.count = bytes;
513
514 rpc_init_task(&data->task, NFS_CLIENT(inode), RPC_TASK_ASYNC,
515 &nfs_write_direct_ops, data);
516 NFS_PROTO(inode)->write_setup(data, FLUSH_STABLE);
517
518 data->task.tk_priority = RPC_PRIORITY_NORMAL;
519 data->task.tk_cookie = (unsigned long) inode;
520
521 lock_kernel();
522 rpc_execute(&data->task);
523 unlock_kernel();
524
525 dfprintk(VFS, "NFS: %4d initiated direct write call (req %s/%Ld, %u bytes @ offset %Lu)\n",
526 data->task.tk_pid,
527 inode->i_sb->s_id,
528 (long long)NFS_FILEID(inode),
529 bytes,
530 (unsigned long long)data->args.offset);
531
532 file_offset += bytes;
533 pgbase += bytes;
534 curpage += pgbase >> PAGE_SHIFT;
535 pgbase &= ~PAGE_MASK;
536
537 count -= bytes;
538 } while (count != 0);
539}
540
Chuck Leverd4cc9482006-03-20 13:44:28 -0500541static ssize_t nfs_direct_write_seg(struct inode *inode, struct nfs_open_context *ctx, unsigned long user_addr, size_t count, loff_t file_offset, struct page **pages, int nr_pages)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700542{
Chuck Lever462d5b32006-03-20 13:44:32 -0500543 ssize_t result;
544 sigset_t oldset;
545 struct rpc_clnt *clnt = NFS_CLIENT(inode);
546 struct nfs_direct_req *dreq;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700547
Chuck Lever462d5b32006-03-20 13:44:32 -0500548 dreq = nfs_direct_write_alloc(count, NFS_SERVER(inode)->wsize);
549 if (!dreq)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700550 return -ENOMEM;
551
Chuck Lever462d5b32006-03-20 13:44:32 -0500552 dreq->pages = pages;
553 dreq->npages = nr_pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700554
555 nfs_begin_data_update(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700556
Chuck Lever462d5b32006-03-20 13:44:32 -0500557 rpc_clnt_sigmask(clnt, &oldset);
558 nfs_direct_write_schedule(dreq, inode, ctx, user_addr, count,
559 file_offset);
560 result = nfs_direct_write_wait(dreq, clnt->cl_intr);
561 rpc_clnt_sigunmask(clnt, &oldset);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700562
Trond Myklebust951a1432005-06-22 17:16:30 +0000563 nfs_end_data_update(inode);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700564
Chuck Lever462d5b32006-03-20 13:44:32 -0500565 return result;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700566}
567
Chuck Leverd4cc9482006-03-20 13:44:28 -0500568/*
Linus Torvalds1da177e2005-04-16 15:20:36 -0700569 * Upon return, generic_file_direct_IO invalidates any cached pages
570 * that non-direct readers might access, so they will pick up these
571 * writes immediately.
572 */
Chuck Leverd4cc9482006-03-20 13:44:28 -0500573static ssize_t nfs_direct_write(struct inode *inode, struct nfs_open_context *ctx, const struct iovec *iov, loff_t file_offset, unsigned long nr_segs)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700574{
575 ssize_t tot_bytes = 0;
576 unsigned long seg = 0;
577
578 while ((seg < nr_segs) && (tot_bytes >= 0)) {
579 ssize_t result;
580 int page_count;
581 struct page **pages;
582 const struct iovec *vec = &iov[seg++];
583 unsigned long user_addr = (unsigned long) vec->iov_base;
584 size_t size = vec->iov_len;
585
586 page_count = nfs_get_user_pages(WRITE, user_addr, size, &pages);
587 if (page_count < 0) {
588 nfs_free_user_pages(pages, 0, 0);
589 if (tot_bytes > 0)
590 break;
591 return page_count;
592 }
593
Chuck Lever91d5b472006-03-20 13:44:14 -0500594 nfs_add_stats(inode, NFSIOS_DIRECTWRITTENBYTES, size);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700595 result = nfs_direct_write_seg(inode, ctx, user_addr, size,
596 file_offset, pages, page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700597
598 if (result <= 0) {
599 if (tot_bytes > 0)
600 break;
601 return result;
602 }
Chuck Lever91d5b472006-03-20 13:44:14 -0500603 nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, result);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700604 tot_bytes += result;
605 file_offset += result;
606 if (result < size)
607 break;
608 }
609 return tot_bytes;
610}
611
612/**
Linus Torvalds1da177e2005-04-16 15:20:36 -0700613 * nfs_file_direct_read - file direct read operation for NFS files
614 * @iocb: target I/O control block
615 * @buf: user's buffer into which to read data
616 * count: number of bytes to read
617 * pos: byte offset in file where reading starts
618 *
619 * We use this function for direct reads instead of calling
620 * generic_file_aio_read() in order to avoid gfar's check to see if
621 * the request starts before the end of the file. For that check
622 * to work, we must generate a GETATTR before each direct read, and
623 * even then there is a window between the GETATTR and the subsequent
624 * READ where the file size could change. So our preference is simply
625 * to do all reads the application wants, and the server will take
626 * care of managing the end of file boundary.
627 *
628 * This function also eliminates unnecessarily updating the file's
629 * atime locally, as the NFS server sets the file's atime, and this
630 * client must read the updated atime from the server back into its
631 * cache.
632 */
Chuck Leverd4cc9482006-03-20 13:44:28 -0500633ssize_t nfs_file_direct_read(struct kiocb *iocb, char __user *buf, size_t count, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700634{
635 ssize_t retval = -EINVAL;
Chuck Lever0cdd80d2006-03-20 13:44:29 -0500636 int page_count;
637 struct page **pages;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700638 struct file *file = iocb->ki_filp;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700639 struct address_space *mapping = file->f_mapping;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700640
Chuck Leverce1a8e62005-11-30 18:08:17 -0500641 dprintk("nfs: direct read(%s/%s, %lu@%Ld)\n",
Chuck Lever0bbacc42005-11-01 16:53:32 -0500642 file->f_dentry->d_parent->d_name.name,
643 file->f_dentry->d_name.name,
Chuck Leverce1a8e62005-11-30 18:08:17 -0500644 (unsigned long) count, (long long) pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700645
Linus Torvalds1da177e2005-04-16 15:20:36 -0700646 if (count < 0)
647 goto out;
648 retval = -EFAULT;
Chuck Lever0cdd80d2006-03-20 13:44:29 -0500649 if (!access_ok(VERIFY_WRITE, buf, count))
Linus Torvalds1da177e2005-04-16 15:20:36 -0700650 goto out;
651 retval = 0;
652 if (!count)
653 goto out;
654
Trond Myklebust29884df2005-12-13 16:13:54 -0500655 retval = nfs_sync_mapping(mapping);
656 if (retval)
657 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700658
Chuck Lever0cdd80d2006-03-20 13:44:29 -0500659 page_count = nfs_get_user_pages(READ, (unsigned long) buf,
660 count, &pages);
661 if (page_count < 0) {
662 nfs_free_user_pages(pages, 0, 0);
663 retval = page_count;
664 goto out;
665 }
666
Chuck Lever99514f82006-03-20 13:44:30 -0500667 retval = nfs_direct_read(iocb, (unsigned long) buf, count, pos,
Chuck Lever0cdd80d2006-03-20 13:44:29 -0500668 pages, page_count);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700669 if (retval > 0)
Chuck Lever0cdd80d2006-03-20 13:44:29 -0500670 iocb->ki_pos = pos + retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700671
672out:
673 return retval;
674}
675
676/**
677 * nfs_file_direct_write - file direct write operation for NFS files
678 * @iocb: target I/O control block
679 * @buf: user's buffer from which to write data
680 * count: number of bytes to write
681 * pos: byte offset in file where writing starts
682 *
683 * We use this function for direct writes instead of calling
684 * generic_file_aio_write() in order to avoid taking the inode
685 * semaphore and updating the i_size. The NFS server will set
686 * the new i_size and this client must read the updated size
687 * back into its cache. We let the server do generic write
688 * parameter checking and report problems.
689 *
690 * We also avoid an unnecessary invocation of generic_osync_inode(),
691 * as it is fairly meaningless to sync the metadata of an NFS file.
692 *
693 * We eliminate local atime updates, see direct read above.
694 *
695 * We avoid unnecessary page cache invalidations for normal cached
696 * readers of this file.
697 *
698 * Note that O_APPEND is not supported for NFS direct writes, as there
699 * is no atomic O_APPEND write facility in the NFS protocol.
700 */
Chuck Leverd4cc9482006-03-20 13:44:28 -0500701ssize_t nfs_file_direct_write(struct kiocb *iocb, const char __user *buf, size_t count, loff_t pos)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700702{
Chuck Leverce1a8e62005-11-30 18:08:17 -0500703 ssize_t retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700704 struct file *file = iocb->ki_filp;
705 struct nfs_open_context *ctx =
706 (struct nfs_open_context *) file->private_data;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700707 struct address_space *mapping = file->f_mapping;
708 struct inode *inode = mapping->host;
709 struct iovec iov = {
710 .iov_base = (char __user *)buf,
Linus Torvalds1da177e2005-04-16 15:20:36 -0700711 };
712
Chuck Leverce1a8e62005-11-30 18:08:17 -0500713 dfprintk(VFS, "nfs: direct write(%s/%s, %lu@%Ld)\n",
Chuck Lever0bbacc42005-11-01 16:53:32 -0500714 file->f_dentry->d_parent->d_name.name,
Chuck Leverce1a8e62005-11-30 18:08:17 -0500715 file->f_dentry->d_name.name,
716 (unsigned long) count, (long long) pos);
Linus Torvalds1da177e2005-04-16 15:20:36 -0700717
Chuck Leverce1a8e62005-11-30 18:08:17 -0500718 retval = -EINVAL;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700719 if (!is_sync_kiocb(iocb))
720 goto out;
Chuck Leverce1a8e62005-11-30 18:08:17 -0500721
722 retval = generic_write_checks(file, &pos, &count, 0);
723 if (retval)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700724 goto out;
Chuck Leverce1a8e62005-11-30 18:08:17 -0500725
726 retval = -EINVAL;
727 if ((ssize_t) count < 0)
Linus Torvalds1da177e2005-04-16 15:20:36 -0700728 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700729 retval = 0;
730 if (!count)
731 goto out;
Chuck Leverce1a8e62005-11-30 18:08:17 -0500732 iov.iov_len = count,
733
734 retval = -EFAULT;
735 if (!access_ok(VERIFY_READ, iov.iov_base, iov.iov_len))
736 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700737
Trond Myklebust29884df2005-12-13 16:13:54 -0500738 retval = nfs_sync_mapping(mapping);
739 if (retval)
740 goto out;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700741
742 retval = nfs_direct_write(inode, ctx, &iov, pos, 1);
743 if (mapping->nrpages)
744 invalidate_inode_pages2(mapping);
745 if (retval > 0)
Chuck Leverce1a8e62005-11-30 18:08:17 -0500746 iocb->ki_pos = pos + retval;
Linus Torvalds1da177e2005-04-16 15:20:36 -0700747
748out:
749 return retval;
750}
751
752int nfs_init_directcache(void)
753{
754 nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
755 sizeof(struct nfs_direct_req),
756 0, SLAB_RECLAIM_ACCOUNT,
757 NULL, NULL);
758 if (nfs_direct_cachep == NULL)
759 return -ENOMEM;
760
761 return 0;
762}
763
764void nfs_destroy_directcache(void)
765{
766 if (kmem_cache_destroy(nfs_direct_cachep))
767 printk(KERN_INFO "nfs_direct_cache: not all structures were freed\n");
768}