Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | #ifndef __LINUX__AIO_H |
| 2 | #define __LINUX__AIO_H |
| 3 | |
| 4 | #include <linux/list.h> |
| 5 | #include <linux/workqueue.h> |
| 6 | #include <linux/aio_abi.h> |
Badari Pulavarty | 027445c | 2006-09-30 23:28:46 -0700 | [diff] [blame] | 7 | #include <linux/uio.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 8 | |
| 9 | #include <asm/atomic.h> |
Badari Pulavarty | eed4e51 | 2006-09-30 23:28:49 -0700 | [diff] [blame] | 10 | #include <linux/uio.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 11 | |
| 12 | #define AIO_MAXSEGS 4 |
| 13 | #define AIO_KIOGRP_NR_ATOMIC 8 |
| 14 | |
| 15 | struct kioctx; |
| 16 | |
| 17 | /* Notes on cancelling a kiocb: |
| 18 | * If a kiocb is cancelled, aio_complete may return 0 to indicate |
| 19 | * that cancel has not yet disposed of the kiocb. All cancel |
| 20 | * operations *must* call aio_put_req to dispose of the kiocb |
| 21 | * to guard against races with the completion code. |
| 22 | */ |
| 23 | #define KIOCB_C_CANCELLED 0x01 |
| 24 | #define KIOCB_C_COMPLETE 0x02 |
| 25 | |
| 26 | #define KIOCB_SYNC_KEY (~0U) |
| 27 | |
| 28 | /* ki_flags bits */ |
Zach Brown | 4faa528 | 2005-10-17 16:43:33 -0700 | [diff] [blame] | 29 | /* |
| 30 | * This may be used for cancel/retry serialization in the future, but |
| 31 | * for now it's unused and we probably don't want modules to even |
| 32 | * think they can use it. |
| 33 | */ |
| 34 | /* #define KIF_LOCKED 0 */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 35 | #define KIF_KICKED 1 |
| 36 | #define KIF_CANCELLED 2 |
| 37 | |
| 38 | #define kiocbTryLock(iocb) test_and_set_bit(KIF_LOCKED, &(iocb)->ki_flags) |
| 39 | #define kiocbTryKick(iocb) test_and_set_bit(KIF_KICKED, &(iocb)->ki_flags) |
| 40 | |
| 41 | #define kiocbSetLocked(iocb) set_bit(KIF_LOCKED, &(iocb)->ki_flags) |
| 42 | #define kiocbSetKicked(iocb) set_bit(KIF_KICKED, &(iocb)->ki_flags) |
| 43 | #define kiocbSetCancelled(iocb) set_bit(KIF_CANCELLED, &(iocb)->ki_flags) |
| 44 | |
| 45 | #define kiocbClearLocked(iocb) clear_bit(KIF_LOCKED, &(iocb)->ki_flags) |
| 46 | #define kiocbClearKicked(iocb) clear_bit(KIF_KICKED, &(iocb)->ki_flags) |
| 47 | #define kiocbClearCancelled(iocb) clear_bit(KIF_CANCELLED, &(iocb)->ki_flags) |
| 48 | |
| 49 | #define kiocbIsLocked(iocb) test_bit(KIF_LOCKED, &(iocb)->ki_flags) |
| 50 | #define kiocbIsKicked(iocb) test_bit(KIF_KICKED, &(iocb)->ki_flags) |
| 51 | #define kiocbIsCancelled(iocb) test_bit(KIF_CANCELLED, &(iocb)->ki_flags) |
| 52 | |
Zach Brown | 897f15f | 2005-09-30 11:58:55 -0700 | [diff] [blame] | 53 | /* is there a better place to document function pointer methods? */ |
| 54 | /** |
| 55 | * ki_retry - iocb forward progress callback |
| 56 | * @kiocb: The kiocb struct to advance by performing an operation. |
| 57 | * |
| 58 | * This callback is called when the AIO core wants a given AIO operation |
| 59 | * to make forward progress. The kiocb argument describes the operation |
| 60 | * that is to be performed. As the operation proceeds, perhaps partially, |
| 61 | * ki_retry is expected to update the kiocb with progress made. Typically |
| 62 | * ki_retry is set in the AIO core and it itself calls file_operations |
| 63 | * helpers. |
| 64 | * |
| 65 | * ki_retry's return value determines when the AIO operation is completed |
| 66 | * and an event is generated in the AIO event ring. Except the special |
| 67 | * return values described below, the value that is returned from ki_retry |
| 68 | * is transferred directly into the completion ring as the operation's |
| 69 | * resulting status. Once this has happened ki_retry *MUST NOT* reference |
| 70 | * the kiocb pointer again. |
| 71 | * |
| 72 | * If ki_retry returns -EIOCBQUEUED it has made a promise that aio_complete() |
| 73 | * will be called on the kiocb pointer in the future. The AIO core will |
| 74 | * not ask the method again -- ki_retry must ensure forward progress. |
| 75 | * aio_complete() must be called once and only once in the future, multiple |
| 76 | * calls may result in undefined behaviour. |
| 77 | * |
| 78 | * If ki_retry returns -EIOCBRETRY it has made a promise that kick_iocb() |
| 79 | * will be called on the kiocb pointer in the future. This may happen |
| 80 | * through generic helpers that associate kiocb->ki_wait with a wait |
| 81 | * queue head that ki_retry uses via current->io_wait. It can also happen |
| 82 | * with custom tracking and manual calls to kick_iocb(), though that is |
| 83 | * discouraged. In either case, kick_iocb() must be called once and only |
| 84 | * once. ki_retry must ensure forward progress, the AIO core will wait |
| 85 | * indefinitely for kick_iocb() to be called. |
| 86 | */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 87 | struct kiocb { |
| 88 | struct list_head ki_run_list; |
| 89 | long ki_flags; |
| 90 | int ki_users; |
| 91 | unsigned ki_key; /* id of this request */ |
| 92 | |
| 93 | struct file *ki_filp; |
| 94 | struct kioctx *ki_ctx; /* may be NULL for sync ops */ |
| 95 | int (*ki_cancel)(struct kiocb *, struct io_event *); |
| 96 | ssize_t (*ki_retry)(struct kiocb *); |
| 97 | void (*ki_dtor)(struct kiocb *); |
| 98 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 99 | union { |
| 100 | void __user *user; |
| 101 | struct task_struct *tsk; |
| 102 | } ki_obj; |
Benjamin LaHaise | 59d9136 | 2006-01-08 01:04:34 -0800 | [diff] [blame] | 103 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 104 | __u64 ki_user_data; /* user's data for completion */ |
Benjamin LaHaise | 59d9136 | 2006-01-08 01:04:34 -0800 | [diff] [blame] | 105 | wait_queue_t ki_wait; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 106 | loff_t ki_pos; |
Benjamin LaHaise | 59d9136 | 2006-01-08 01:04:34 -0800 | [diff] [blame] | 107 | |
Chen, Kenneth W | e61c901 | 2006-12-13 00:34:36 -0800 | [diff] [blame] | 108 | atomic_t ki_bio_count; /* num bio used for this iocb */ |
Benjamin LaHaise | 59d9136 | 2006-01-08 01:04:34 -0800 | [diff] [blame] | 109 | void *private; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 110 | /* State that we remember to be able to restart/retry */ |
| 111 | unsigned short ki_opcode; |
| 112 | size_t ki_nbytes; /* copy of iocb->aio_nbytes */ |
| 113 | char __user *ki_buf; /* remaining iocb->aio_buf */ |
| 114 | size_t ki_left; /* remaining bytes */ |
Badari Pulavarty | 027445c | 2006-09-30 23:28:46 -0700 | [diff] [blame] | 115 | struct iovec ki_inline_vec; /* inline vector */ |
Badari Pulavarty | eed4e51 | 2006-09-30 23:28:49 -0700 | [diff] [blame] | 116 | struct iovec *ki_iovec; |
| 117 | unsigned long ki_nr_segs; |
| 118 | unsigned long ki_cur_seg; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 119 | |
Benjamin LaHaise | 59d9136 | 2006-01-08 01:04:34 -0800 | [diff] [blame] | 120 | struct list_head ki_list; /* the aio core uses this |
| 121 | * for cancellation */ |
Davide Libenzi | 9c3060b | 2007-05-10 22:23:21 -0700 | [diff] [blame] | 122 | |
| 123 | /* |
| 124 | * If the aio_resfd field of the userspace iocb is not zero, |
| 125 | * this is the underlying file* to deliver event to. |
| 126 | */ |
| 127 | struct file *ki_eventfd; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 128 | }; |
| 129 | |
| 130 | #define is_sync_kiocb(iocb) ((iocb)->ki_key == KIOCB_SYNC_KEY) |
| 131 | #define init_sync_kiocb(x, filp) \ |
| 132 | do { \ |
| 133 | struct task_struct *tsk = current; \ |
| 134 | (x)->ki_flags = 0; \ |
| 135 | (x)->ki_users = 1; \ |
| 136 | (x)->ki_key = KIOCB_SYNC_KEY; \ |
| 137 | (x)->ki_filp = (filp); \ |
Zach Brown | 20dcae3 | 2005-11-13 16:07:33 -0800 | [diff] [blame] | 138 | (x)->ki_ctx = NULL; \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 139 | (x)->ki_cancel = NULL; \ |
Benjamin LaHaise | 59d9136 | 2006-01-08 01:04:34 -0800 | [diff] [blame] | 140 | (x)->ki_retry = NULL; \ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 141 | (x)->ki_dtor = NULL; \ |
| 142 | (x)->ki_obj.tsk = tsk; \ |
| 143 | (x)->ki_user_data = 0; \ |
| 144 | init_wait((&(x)->ki_wait)); \ |
| 145 | } while (0) |
| 146 | |
| 147 | #define AIO_RING_MAGIC 0xa10a10a1 |
| 148 | #define AIO_RING_COMPAT_FEATURES 1 |
| 149 | #define AIO_RING_INCOMPAT_FEATURES 0 |
| 150 | struct aio_ring { |
| 151 | unsigned id; /* kernel internal index number */ |
| 152 | unsigned nr; /* number of io_events */ |
| 153 | unsigned head; |
| 154 | unsigned tail; |
| 155 | |
| 156 | unsigned magic; |
| 157 | unsigned compat_features; |
| 158 | unsigned incompat_features; |
| 159 | unsigned header_length; /* size of aio_ring */ |
| 160 | |
| 161 | |
| 162 | struct io_event io_events[0]; |
| 163 | }; /* 128 bytes + ring size */ |
| 164 | |
| 165 | #define aio_ring_avail(info, ring) (((ring)->head + (info)->nr - 1 - (ring)->tail) % (info)->nr) |
| 166 | |
| 167 | #define AIO_RING_PAGES 8 |
| 168 | struct aio_ring_info { |
| 169 | unsigned long mmap_base; |
| 170 | unsigned long mmap_size; |
| 171 | |
| 172 | struct page **ring_pages; |
| 173 | spinlock_t ring_lock; |
| 174 | long nr_pages; |
| 175 | |
| 176 | unsigned nr, tail; |
| 177 | |
| 178 | struct page *internal_pages[AIO_RING_PAGES]; |
| 179 | }; |
| 180 | |
| 181 | struct kioctx { |
| 182 | atomic_t users; |
| 183 | int dead; |
| 184 | struct mm_struct *mm; |
| 185 | |
| 186 | /* This needs improving */ |
| 187 | unsigned long user_id; |
| 188 | struct kioctx *next; |
| 189 | |
| 190 | wait_queue_head_t wait; |
| 191 | |
| 192 | spinlock_t ctx_lock; |
| 193 | |
| 194 | int reqs_active; |
| 195 | struct list_head active_reqs; /* used for cancellation */ |
| 196 | struct list_head run_list; /* used for kicked reqs */ |
| 197 | |
Zach Brown | d55b5fd | 2005-11-07 00:59:31 -0800 | [diff] [blame] | 198 | /* sys_io_setup currently limits this to an unsigned int */ |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 199 | unsigned max_reqs; |
| 200 | |
| 201 | struct aio_ring_info ring_info; |
| 202 | |
David Howells | 52bad64 | 2006-11-22 14:54:01 +0000 | [diff] [blame] | 203 | struct delayed_work wq; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 204 | }; |
| 205 | |
| 206 | /* prototypes */ |
| 207 | extern unsigned aio_max_size; |
| 208 | |
| 209 | extern ssize_t FASTCALL(wait_on_sync_kiocb(struct kiocb *iocb)); |
| 210 | extern int FASTCALL(aio_put_req(struct kiocb *iocb)); |
| 211 | extern void FASTCALL(kick_iocb(struct kiocb *iocb)); |
| 212 | extern int FASTCALL(aio_complete(struct kiocb *iocb, long res, long res2)); |
| 213 | extern void FASTCALL(__put_ioctx(struct kioctx *ctx)); |
| 214 | struct mm_struct; |
| 215 | extern void FASTCALL(exit_aio(struct mm_struct *mm)); |
| 216 | extern struct kioctx *lookup_ioctx(unsigned long ctx_id); |
| 217 | extern int FASTCALL(io_submit_one(struct kioctx *ctx, |
| 218 | struct iocb __user *user_iocb, struct iocb *iocb)); |
| 219 | |
| 220 | /* semi private, but used by the 32bit emulations: */ |
| 221 | struct kioctx *lookup_ioctx(unsigned long ctx_id); |
| 222 | int FASTCALL(io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, |
| 223 | struct iocb *iocb)); |
| 224 | |
Zach Brown | 5ef1c49 | 2005-11-13 16:07:35 -0800 | [diff] [blame] | 225 | #define get_ioctx(kioctx) do { \ |
Rolf Eike Beer | 3a27111 | 2006-09-30 23:28:09 -0700 | [diff] [blame] | 226 | BUG_ON(atomic_read(&(kioctx)->users) <= 0); \ |
Zach Brown | 5ef1c49 | 2005-11-13 16:07:35 -0800 | [diff] [blame] | 227 | atomic_inc(&(kioctx)->users); \ |
| 228 | } while (0) |
| 229 | #define put_ioctx(kioctx) do { \ |
Rolf Eike Beer | 3a27111 | 2006-09-30 23:28:09 -0700 | [diff] [blame] | 230 | BUG_ON(atomic_read(&(kioctx)->users) <= 0); \ |
Zach Brown | 5ef1c49 | 2005-11-13 16:07:35 -0800 | [diff] [blame] | 231 | if (unlikely(atomic_dec_and_test(&(kioctx)->users))) \ |
| 232 | __put_ioctx(kioctx); \ |
| 233 | } while (0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 234 | |
Andrew Morton | b8522ea | 2007-05-09 02:34:58 -0700 | [diff] [blame] | 235 | #define in_aio() (unlikely(!is_sync_wait(current->io_wait))) |
| 236 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 237 | /* may be used for debugging */ |
| 238 | #define warn_if_async() \ |
| 239 | do { \ |
| 240 | if (in_aio()) { \ |
| 241 | printk(KERN_ERR "%s(%s:%d) called in async context!\n", \ |
| 242 | __FUNCTION__, __FILE__, __LINE__); \ |
| 243 | dump_stack(); \ |
| 244 | } \ |
| 245 | } while (0) |
| 246 | |
| 247 | #define io_wait_to_kiocb(wait) container_of(wait, struct kiocb, ki_wait) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 248 | |
| 249 | #include <linux/aio_abi.h> |
| 250 | |
| 251 | static inline struct kiocb *list_kiocb(struct list_head *h) |
| 252 | { |
| 253 | return list_entry(h, struct kiocb, ki_list); |
| 254 | } |
| 255 | |
| 256 | /* for sysctl: */ |
Zach Brown | d55b5fd | 2005-11-07 00:59:31 -0800 | [diff] [blame] | 257 | extern unsigned long aio_nr; |
| 258 | extern unsigned long aio_max_nr; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 259 | |
| 260 | #endif /* __LINUX__AIO_H */ |