Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 1 | /* |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 2 | * Copyright (C) 2004 - 2007 Jeff Dike (jdike@{addtoit,linux.intel}.com) |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 3 | * Licensed under the GPL |
| 4 | */ |
| 5 | |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 6 | #include <unistd.h> |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 7 | #include <sched.h> |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 8 | #include <signal.h> |
| 9 | #include <errno.h> |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 10 | #include <sys/time.h> |
| 11 | #include <asm/unistd.h> |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 12 | #include "aio.h" |
| 13 | #include "init.h" |
Jeff Dike | da3e30e | 2007-07-23 18:43:47 -0700 | [diff] [blame] | 14 | #include "kern_constants.h" |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 15 | #include "os.h" |
| 16 | #include "user.h" |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 17 | |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 18 | struct aio_thread_req { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 19 | enum aio_type type; |
| 20 | int io_fd; |
| 21 | unsigned long long offset; |
| 22 | char *buf; |
| 23 | int len; |
| 24 | struct aio_context *aio; |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 25 | }; |
| 26 | |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 27 | #if defined(HAVE_AIO_ABI) |
| 28 | #include <linux/aio_abi.h> |
| 29 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 30 | /* |
| 31 | * If we have the headers, we are going to build with AIO enabled. |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 32 | * If we don't have aio in libc, we define the necessary stubs here. |
| 33 | */ |
| 34 | |
| 35 | #if !defined(HAVE_AIO_LIBC) |
| 36 | |
| 37 | static long io_setup(int n, aio_context_t *ctxp) |
| 38 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 39 | return syscall(__NR_io_setup, n, ctxp); |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 40 | } |
| 41 | |
| 42 | static long io_submit(aio_context_t ctx, long nr, struct iocb **iocbpp) |
| 43 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 44 | return syscall(__NR_io_submit, ctx, nr, iocbpp); |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 45 | } |
| 46 | |
| 47 | static long io_getevents(aio_context_t ctx_id, long min_nr, long nr, |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 48 | struct io_event *events, struct timespec *timeout) |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 49 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 50 | return syscall(__NR_io_getevents, ctx_id, min_nr, nr, events, timeout); |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 51 | } |
| 52 | |
| 53 | #endif |
| 54 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 55 | /* |
| 56 | * The AIO_MMAP cases force the mmapped page into memory here |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 57 | * rather than in whatever place first touches the data. I used |
| 58 | * to do this by touching the page, but that's delicate because |
| 59 | * gcc is prone to optimizing that away. So, what's done here |
| 60 | * is we read from the descriptor from which the page was |
| 61 | * mapped. The caller is required to pass an offset which is |
| 62 | * inside the page that was mapped. Thus, when the read |
| 63 | * returns, we know that the page is in the page cache, and |
| 64 | * that it now backs the mmapped area. |
| 65 | */ |
| 66 | |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 67 | static int do_aio(aio_context_t ctx, enum aio_type type, int fd, char *buf, |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 68 | int len, unsigned long long offset, struct aio_context *aio) |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 69 | { |
Jeff Dike | da3e30e | 2007-07-23 18:43:47 -0700 | [diff] [blame] | 70 | struct iocb *iocbp = & ((struct iocb) { |
| 71 | .aio_data = (unsigned long) aio, |
| 72 | .aio_fildes = fd, |
| 73 | .aio_buf = (unsigned long) buf, |
| 74 | .aio_nbytes = len, |
| 75 | .aio_offset = offset |
| 76 | }); |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 77 | char c; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 78 | |
Jeff Dike | da3e30e | 2007-07-23 18:43:47 -0700 | [diff] [blame] | 79 | switch (type) { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 80 | case AIO_READ: |
Jeff Dike | da3e30e | 2007-07-23 18:43:47 -0700 | [diff] [blame] | 81 | iocbp->aio_lio_opcode = IOCB_CMD_PREAD; |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 82 | break; |
| 83 | case AIO_WRITE: |
Jeff Dike | da3e30e | 2007-07-23 18:43:47 -0700 | [diff] [blame] | 84 | iocbp->aio_lio_opcode = IOCB_CMD_PWRITE; |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 85 | break; |
| 86 | case AIO_MMAP: |
Jeff Dike | da3e30e | 2007-07-23 18:43:47 -0700 | [diff] [blame] | 87 | iocbp->aio_lio_opcode = IOCB_CMD_PREAD; |
| 88 | iocbp->aio_buf = (unsigned long) &c; |
| 89 | iocbp->aio_nbytes = sizeof(c); |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 90 | break; |
| 91 | default: |
Jeff Dike | da3e30e | 2007-07-23 18:43:47 -0700 | [diff] [blame] | 92 | printk(UM_KERN_ERR "Bogus op in do_aio - %d\n", type); |
| 93 | return -EINVAL; |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 94 | } |
Jeff Dike | 09ace81 | 2005-09-03 15:57:46 -0700 | [diff] [blame] | 95 | |
Jeff Dike | da3e30e | 2007-07-23 18:43:47 -0700 | [diff] [blame] | 96 | return (io_submit(ctx, 1, &iocbp) > 0) ? 0 : -errno; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 97 | } |
| 98 | |
Jeff Dike | 9683da9 | 2007-02-10 01:44:27 -0800 | [diff] [blame] | 99 | /* Initialized in an initcall and unchanged thereafter */ |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 100 | static aio_context_t ctx = 0; |
| 101 | |
| 102 | static int aio_thread(void *arg) |
| 103 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 104 | struct aio_thread_reply reply; |
| 105 | struct io_event event; |
| 106 | int err, n, reply_fd; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 107 | |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 108 | signal(SIGWINCH, SIG_IGN); |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 109 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 110 | while (1) { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 111 | n = io_getevents(ctx, 1, 1, &event, NULL); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 112 | if (n < 0) { |
| 113 | if (errno == EINTR) |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 114 | continue; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 115 | printk(UM_KERN_ERR "aio_thread - io_getevents failed, " |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 116 | "errno = %d\n", errno); |
| 117 | } |
| 118 | else { |
| 119 | reply = ((struct aio_thread_reply) |
| 120 | { .data = (void *) (long) event.data, |
| 121 | .err = event.res }); |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 122 | reply_fd = ((struct aio_context *) reply.data)->reply_fd; |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 123 | err = write(reply_fd, &reply, sizeof(reply)); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 124 | if (err != sizeof(reply)) |
| 125 | printk(UM_KERN_ERR "aio_thread - write failed, " |
| 126 | "fd = %d, err = %d\n", reply_fd, errno); |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 127 | } |
| 128 | } |
| 129 | return 0; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 130 | } |
| 131 | |
| 132 | #endif |
| 133 | |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 134 | static int do_not_aio(struct aio_thread_req *req) |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 135 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 136 | char c; |
Jeff Dike | ef0470c | 2007-05-06 14:51:33 -0700 | [diff] [blame] | 137 | unsigned long long actual; |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 138 | int n; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 139 | |
Jeff Dike | ef0470c | 2007-05-06 14:51:33 -0700 | [diff] [blame] | 140 | actual = lseek64(req->io_fd, req->offset, SEEK_SET); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 141 | if (actual != req->offset) |
Jeff Dike | ef0470c | 2007-05-06 14:51:33 -0700 | [diff] [blame] | 142 | return -errno; |
| 143 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 144 | switch(req->type) { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 145 | case AIO_READ: |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 146 | n = read(req->io_fd, req->buf, req->len); |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 147 | break; |
| 148 | case AIO_WRITE: |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 149 | n = write(req->io_fd, req->buf, req->len); |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 150 | break; |
| 151 | case AIO_MMAP: |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 152 | n = read(req->io_fd, &c, sizeof(c)); |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 153 | break; |
| 154 | default: |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 155 | printk(UM_KERN_ERR "do_not_aio - bad request type : %d\n", |
| 156 | req->type); |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 157 | return -EINVAL; |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 158 | } |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 159 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 160 | if (n < 0) |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 161 | return -errno; |
| 162 | return 0; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 163 | } |
| 164 | |
Jeff Dike | 9683da9 | 2007-02-10 01:44:27 -0800 | [diff] [blame] | 165 | /* These are initialized in initcalls and not changed */ |
| 166 | static int aio_req_fd_r = -1; |
| 167 | static int aio_req_fd_w = -1; |
| 168 | static int aio_pid = -1; |
Jeff Dike | c439901 | 2007-07-15 23:38:56 -0700 | [diff] [blame] | 169 | static unsigned long aio_stack; |
Jeff Dike | 9683da9 | 2007-02-10 01:44:27 -0800 | [diff] [blame] | 170 | |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 171 | static int not_aio_thread(void *arg) |
| 172 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 173 | struct aio_thread_req req; |
| 174 | struct aio_thread_reply reply; |
| 175 | int err; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 176 | |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 177 | signal(SIGWINCH, SIG_IGN); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 178 | while (1) { |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 179 | err = read(aio_req_fd_r, &req, sizeof(req)); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 180 | if (err != sizeof(req)) { |
| 181 | if (err < 0) |
| 182 | printk(UM_KERN_ERR "not_aio_thread - " |
| 183 | "read failed, fd = %d, err = %d\n", |
| 184 | aio_req_fd_r, |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 185 | errno); |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 186 | else { |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 187 | printk(UM_KERN_ERR "not_aio_thread - short " |
| 188 | "read, fd = %d, length = %d\n", |
| 189 | aio_req_fd_r, err); |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 190 | } |
| 191 | continue; |
| 192 | } |
| 193 | err = do_not_aio(&req); |
| 194 | reply = ((struct aio_thread_reply) { .data = req.aio, |
Jeff Dike | ef0470c | 2007-05-06 14:51:33 -0700 | [diff] [blame] | 195 | .err = err }); |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 196 | err = write(req.aio->reply_fd, &reply, sizeof(reply)); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 197 | if (err != sizeof(reply)) |
| 198 | printk(UM_KERN_ERR "not_aio_thread - write failed, " |
| 199 | "fd = %d, err = %d\n", req.aio->reply_fd, errno); |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 200 | } |
Jeff Dike | 1b57e9c | 2006-01-06 00:18:49 -0800 | [diff] [blame] | 201 | |
| 202 | return 0; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 203 | } |
| 204 | |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 205 | static int init_aio_24(void) |
| 206 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 207 | int fds[2], err; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 208 | |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 209 | err = os_pipe(fds, 1, 1); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 210 | if (err) |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 211 | goto out; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 212 | |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 213 | aio_req_fd_w = fds[0]; |
| 214 | aio_req_fd_r = fds[1]; |
Jeff Dike | 8603ec8 | 2007-05-06 14:51:44 -0700 | [diff] [blame] | 215 | |
| 216 | err = os_set_fd_block(aio_req_fd_w, 0); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 217 | if (err) |
Jeff Dike | 8603ec8 | 2007-05-06 14:51:44 -0700 | [diff] [blame] | 218 | goto out_close_pipe; |
| 219 | |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 220 | err = run_helper_thread(not_aio_thread, NULL, |
Stanislaw Gruszka | 4dbed85 | 2007-12-17 16:19:46 -0800 | [diff] [blame^] | 221 | CLONE_FILES | CLONE_VM, &aio_stack); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 222 | if (err < 0) |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 223 | goto out_close_pipe; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 224 | |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 225 | aio_pid = err; |
| 226 | goto out; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 227 | |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 228 | out_close_pipe: |
Jeff Dike | 512b6fb | 2007-10-16 01:27:11 -0700 | [diff] [blame] | 229 | close(fds[0]); |
| 230 | close(fds[1]); |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 231 | aio_req_fd_w = -1; |
| 232 | aio_req_fd_r = -1; |
| 233 | out: |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 234 | #ifndef HAVE_AIO_ABI |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 235 | printk(UM_KERN_INFO "/usr/include/linux/aio_abi.h not present during " |
| 236 | "build\n"); |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 237 | #endif |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 238 | printk(UM_KERN_INFO "2.6 host AIO support not used - falling back to " |
| 239 | "I/O thread\n"); |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 240 | return 0; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 241 | } |
| 242 | |
| 243 | #ifdef HAVE_AIO_ABI |
| 244 | #define DEFAULT_24_AIO 0 |
| 245 | static int init_aio_26(void) |
| 246 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 247 | int err; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 248 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 249 | if (io_setup(256, &ctx)) { |
Jeff Dike | b4fd310 | 2005-09-16 19:27:49 -0700 | [diff] [blame] | 250 | err = -errno; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 251 | printk(UM_KERN_ERR "aio_thread failed to initialize context, " |
| 252 | "err = %d\n", errno); |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 253 | return err; |
| 254 | } |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 255 | |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 256 | err = run_helper_thread(aio_thread, NULL, |
Stanislaw Gruszka | 4dbed85 | 2007-12-17 16:19:46 -0800 | [diff] [blame^] | 257 | CLONE_FILES | CLONE_VM, &aio_stack); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 258 | if (err < 0) |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 259 | return err; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 260 | |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 261 | aio_pid = err; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 262 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 263 | printk(UM_KERN_INFO "Using 2.6 host AIO\n"); |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 264 | return 0; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 265 | } |
| 266 | |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 267 | static int submit_aio_26(enum aio_type type, int io_fd, char *buf, int len, |
| 268 | unsigned long long offset, struct aio_context *aio) |
| 269 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 270 | struct aio_thread_reply reply; |
| 271 | int err; |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 272 | |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 273 | err = do_aio(ctx, type, io_fd, buf, len, offset, aio); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 274 | if (err) { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 275 | reply = ((struct aio_thread_reply) { .data = aio, |
| 276 | .err = err }); |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 277 | err = write(aio->reply_fd, &reply, sizeof(reply)); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 278 | if (err != sizeof(reply)) { |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 279 | err = -errno; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 280 | printk(UM_KERN_ERR "submit_aio_26 - write failed, " |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 281 | "fd = %d, err = %d\n", aio->reply_fd, -err); |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 282 | } |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 283 | else err = 0; |
| 284 | } |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 285 | |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 286 | return err; |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 287 | } |
| 288 | |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 289 | #else |
| 290 | #define DEFAULT_24_AIO 1 |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 291 | static int init_aio_26(void) |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 292 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 293 | return -ENOSYS; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 294 | } |
| 295 | |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 296 | static int submit_aio_26(enum aio_type type, int io_fd, char *buf, int len, |
| 297 | unsigned long long offset, struct aio_context *aio) |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 298 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 299 | return -ENOSYS; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 300 | } |
| 301 | #endif |
| 302 | |
Jeff Dike | 9683da9 | 2007-02-10 01:44:27 -0800 | [diff] [blame] | 303 | /* Initialized in an initcall and unchanged thereafter */ |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 304 | static int aio_24 = DEFAULT_24_AIO; |
| 305 | |
| 306 | static int __init set_aio_24(char *name, int *add) |
| 307 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 308 | aio_24 = 1; |
| 309 | return 0; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 310 | } |
| 311 | |
| 312 | __uml_setup("aio=2.4", set_aio_24, |
| 313 | "aio=2.4\n" |
| 314 | " This is used to force UML to use 2.4-style AIO even when 2.6 AIO is\n" |
| 315 | " available. 2.4 AIO is a single thread that handles one request at a\n" |
| 316 | " time, synchronously. 2.6 AIO is a thread which uses the 2.6 AIO \n" |
| 317 | " interface to handle an arbitrary number of pending requests. 2.6 AIO \n" |
| 318 | " is not available in tt mode, on 2.4 hosts, or when UML is built with\n" |
| 319 | " /usr/include/linux/aio_abi.h not available. Many distributions don't\n" |
| 320 | " include aio_abi.h, so you will need to copy it from a kernel tree to\n" |
| 321 | " your /usr/include/linux in order to build an AIO-capable UML\n\n" |
| 322 | ); |
| 323 | |
| 324 | static int init_aio(void) |
| 325 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 326 | int err; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 327 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 328 | if (!aio_24) { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 329 | err = init_aio_26(); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 330 | if (err && (errno == ENOSYS)) { |
| 331 | printk(UM_KERN_INFO "2.6 AIO not supported on the " |
| 332 | "host - reverting to 2.4 AIO\n"); |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 333 | aio_24 = 1; |
| 334 | } |
| 335 | else return err; |
| 336 | } |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 337 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 338 | if (aio_24) |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 339 | return init_aio_24(); |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 340 | |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 341 | return 0; |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 342 | } |
| 343 | |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 344 | /* |
| 345 | * The reason for the __initcall/__uml_exitcall asymmetry is that init_aio |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 346 | * needs to be called when the kernel is running because it calls run_helper, |
| 347 | * which needs get_free_page. exit_aio is a __uml_exitcall because the generic |
| 348 | * kernel does not run __exitcalls on shutdown, and can't because many of them |
| 349 | * break when called outside of module unloading. |
| 350 | */ |
| 351 | __initcall(init_aio); |
| 352 | |
| 353 | static void exit_aio(void) |
| 354 | { |
Jeff Dike | c439901 | 2007-07-15 23:38:56 -0700 | [diff] [blame] | 355 | if (aio_pid != -1) { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 356 | os_kill_process(aio_pid, 1); |
Jeff Dike | c439901 | 2007-07-15 23:38:56 -0700 | [diff] [blame] | 357 | free_stack(aio_stack, 0); |
| 358 | } |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 359 | } |
| 360 | |
| 361 | __uml_exitcall(exit_aio); |
| 362 | |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 363 | static int submit_aio_24(enum aio_type type, int io_fd, char *buf, int len, |
| 364 | unsigned long long offset, struct aio_context *aio) |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 365 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 366 | struct aio_thread_req req = { .type = type, |
| 367 | .io_fd = io_fd, |
| 368 | .offset = offset, |
| 369 | .buf = buf, |
| 370 | .len = len, |
| 371 | .aio = aio, |
| 372 | }; |
| 373 | int err; |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 374 | |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 375 | err = write(aio_req_fd_w, &req, sizeof(req)); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 376 | if (err == sizeof(req)) |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 377 | err = 0; |
Jeff Dike | a61f334 | 2007-05-06 14:51:35 -0700 | [diff] [blame] | 378 | else err = -errno; |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 379 | |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 380 | return err; |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 381 | } |
| 382 | |
| 383 | int submit_aio(enum aio_type type, int io_fd, char *buf, int len, |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 384 | unsigned long long offset, int reply_fd, |
| 385 | struct aio_context *aio) |
Jeff Dike | 91acb21 | 2005-10-10 23:10:32 -0400 | [diff] [blame] | 386 | { |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 387 | aio->reply_fd = reply_fd; |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 388 | if (aio_24) |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 389 | return submit_aio_24(type, io_fd, buf, len, offset, aio); |
Jeff Dike | ba180fd | 2007-10-16 01:27:00 -0700 | [diff] [blame] | 390 | else |
Jeff Dike | d50084a | 2006-01-06 00:18:50 -0800 | [diff] [blame] | 391 | return submit_aio_26(type, io_fd, buf, len, offset, aio); |
Jeff Dike | 75e5584 | 2005-09-03 15:57:45 -0700 | [diff] [blame] | 392 | } |