Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 1 | /* |
| 2 | * linux/fs/file.c |
| 3 | * |
| 4 | * Copyright (C) 1998-1999, Stephen Tweedie and Bill Hawes |
| 5 | * |
| 6 | * Manage the dynamic fd arrays in the process files_struct. |
| 7 | */ |
| 8 | |
| 9 | #include <linux/fs.h> |
| 10 | #include <linux/mm.h> |
| 11 | #include <linux/time.h> |
| 12 | #include <linux/slab.h> |
| 13 | #include <linux/vmalloc.h> |
| 14 | #include <linux/file.h> |
Al Viro | 9f3acc3 | 2008-04-24 07:44:08 -0400 | [diff] [blame^] | 15 | #include <linux/fdtable.h> |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 16 | #include <linux/bitops.h> |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 17 | #include <linux/interrupt.h> |
| 18 | #include <linux/spinlock.h> |
| 19 | #include <linux/rcupdate.h> |
| 20 | #include <linux/workqueue.h> |
| 21 | |
| 22 | struct fdtable_defer { |
| 23 | spinlock_t lock; |
| 24 | struct work_struct wq; |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 25 | struct fdtable *next; |
| 26 | }; |
| 27 | |
Eric Dumazet | 9cfe015 | 2008-02-06 01:37:16 -0800 | [diff] [blame] | 28 | int sysctl_nr_open __read_mostly = 1024*1024; |
| 29 | |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 30 | /* |
| 31 | * We use this list to defer free fdtables that have vmalloced |
| 32 | * sets/arrays. By keeping a per-cpu list, we avoid having to embed |
| 33 | * the work_struct in fdtable itself which avoids a 64 byte (i386) increase in |
| 34 | * this per-task structure. |
| 35 | */ |
| 36 | static DEFINE_PER_CPU(struct fdtable_defer, fdtable_defer_list); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 37 | |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 38 | static inline void * alloc_fdmem(unsigned int size) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 39 | { |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 40 | if (size <= PAGE_SIZE) |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 41 | return kmalloc(size, GFP_KERNEL); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 42 | else |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 43 | return vmalloc(size); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 44 | } |
| 45 | |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 46 | static inline void free_fdarr(struct fdtable *fdt) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 47 | { |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 48 | if (fdt->max_fds <= (PAGE_SIZE / sizeof(struct file *))) |
| 49 | kfree(fdt->fd); |
| 50 | else |
| 51 | vfree(fdt->fd); |
| 52 | } |
| 53 | |
| 54 | static inline void free_fdset(struct fdtable *fdt) |
| 55 | { |
| 56 | if (fdt->max_fds <= (PAGE_SIZE * BITS_PER_BYTE / 2)) |
| 57 | kfree(fdt->open_fds); |
| 58 | else |
| 59 | vfree(fdt->open_fds); |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 60 | } |
| 61 | |
David Howells | 65f27f3 | 2006-11-22 14:55:48 +0000 | [diff] [blame] | 62 | static void free_fdtable_work(struct work_struct *work) |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 63 | { |
David Howells | 65f27f3 | 2006-11-22 14:55:48 +0000 | [diff] [blame] | 64 | struct fdtable_defer *f = |
| 65 | container_of(work, struct fdtable_defer, wq); |
Dipankar Sarma | badf166 | 2005-09-09 13:04:10 -0700 | [diff] [blame] | 66 | struct fdtable *fdt; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 67 | |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 68 | spin_lock_bh(&f->lock); |
| 69 | fdt = f->next; |
| 70 | f->next = NULL; |
| 71 | spin_unlock_bh(&f->lock); |
| 72 | while(fdt) { |
| 73 | struct fdtable *next = fdt->next; |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 74 | vfree(fdt->fd); |
| 75 | free_fdset(fdt); |
| 76 | kfree(fdt); |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 77 | fdt = next; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 78 | } |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 79 | } |
| 80 | |
Vadim Lobanov | 4fd4581 | 2006-12-10 02:21:17 -0800 | [diff] [blame] | 81 | void free_fdtable_rcu(struct rcu_head *rcu) |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 82 | { |
| 83 | struct fdtable *fdt = container_of(rcu, struct fdtable, rcu); |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 84 | struct fdtable_defer *fddef; |
| 85 | |
| 86 | BUG_ON(!fdt); |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 87 | |
Vadim Lobanov | 4fd4581 | 2006-12-10 02:21:17 -0800 | [diff] [blame] | 88 | if (fdt->max_fds <= NR_OPEN_DEFAULT) { |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 89 | /* |
Vadim Lobanov | 4fd4581 | 2006-12-10 02:21:17 -0800 | [diff] [blame] | 90 | * This fdtable is embedded in the files structure and that |
| 91 | * structure itself is getting destroyed. |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 92 | */ |
Vadim Lobanov | 4fd4581 | 2006-12-10 02:21:17 -0800 | [diff] [blame] | 93 | kmem_cache_free(files_cachep, |
| 94 | container_of(fdt, struct files_struct, fdtab)); |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 95 | return; |
| 96 | } |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 97 | if (fdt->max_fds <= (PAGE_SIZE / sizeof(struct file *))) { |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 98 | kfree(fdt->fd); |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 99 | kfree(fdt->open_fds); |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 100 | kfree(fdt); |
| 101 | } else { |
| 102 | fddef = &get_cpu_var(fdtable_defer_list); |
| 103 | spin_lock(&fddef->lock); |
| 104 | fdt->next = fddef->next; |
| 105 | fddef->next = fdt; |
Tejun Heo | 593be07 | 2006-12-06 20:36:01 -0800 | [diff] [blame] | 106 | /* vmallocs are handled from the workqueue context */ |
| 107 | schedule_work(&fddef->wq); |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 108 | spin_unlock(&fddef->lock); |
| 109 | put_cpu_var(fdtable_defer_list); |
| 110 | } |
| 111 | } |
| 112 | |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 113 | /* |
| 114 | * Expand the fdset in the files_struct. Called with the files spinlock |
| 115 | * held for write. |
| 116 | */ |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 117 | static void copy_fdtable(struct fdtable *nfdt, struct fdtable *ofdt) |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 118 | { |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 119 | unsigned int cpy, set; |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 120 | |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 121 | BUG_ON(nfdt->max_fds < ofdt->max_fds); |
| 122 | if (ofdt->max_fds == 0) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 123 | return; |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 124 | |
| 125 | cpy = ofdt->max_fds * sizeof(struct file *); |
| 126 | set = (nfdt->max_fds - ofdt->max_fds) * sizeof(struct file *); |
| 127 | memcpy(nfdt->fd, ofdt->fd, cpy); |
| 128 | memset((char *)(nfdt->fd) + cpy, 0, set); |
| 129 | |
| 130 | cpy = ofdt->max_fds / BITS_PER_BYTE; |
| 131 | set = (nfdt->max_fds - ofdt->max_fds) / BITS_PER_BYTE; |
| 132 | memcpy(nfdt->open_fds, ofdt->open_fds, cpy); |
| 133 | memset((char *)(nfdt->open_fds) + cpy, 0, set); |
| 134 | memcpy(nfdt->close_on_exec, ofdt->close_on_exec, cpy); |
| 135 | memset((char *)(nfdt->close_on_exec) + cpy, 0, set); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 136 | } |
| 137 | |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 138 | static struct fdtable * alloc_fdtable(unsigned int nr) |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 139 | { |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 140 | struct fdtable *fdt; |
| 141 | char *data; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 142 | |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 143 | /* |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 144 | * Figure out how many fds we actually want to support in this fdtable. |
| 145 | * Allocation steps are keyed to the size of the fdarray, since it |
| 146 | * grows far faster than any of the other dynamic data. We try to fit |
| 147 | * the fdarray into comfortable page-tuned chunks: starting at 1024B |
| 148 | * and growing in powers of two from there on. |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 149 | */ |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 150 | nr /= (1024 / sizeof(struct file *)); |
| 151 | nr = roundup_pow_of_two(nr + 1); |
| 152 | nr *= (1024 / sizeof(struct file *)); |
Eric Dumazet | 9cfe015 | 2008-02-06 01:37:16 -0800 | [diff] [blame] | 153 | if (nr > sysctl_nr_open) |
| 154 | nr = sysctl_nr_open; |
Vadim Lobanov | bbea9f6 | 2006-12-10 02:21:12 -0800 | [diff] [blame] | 155 | |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 156 | fdt = kmalloc(sizeof(struct fdtable), GFP_KERNEL); |
| 157 | if (!fdt) |
Vadim Lobanov | bbea9f6 | 2006-12-10 02:21:12 -0800 | [diff] [blame] | 158 | goto out; |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 159 | fdt->max_fds = nr; |
| 160 | data = alloc_fdmem(nr * sizeof(struct file *)); |
| 161 | if (!data) |
| 162 | goto out_fdt; |
| 163 | fdt->fd = (struct file **)data; |
| 164 | data = alloc_fdmem(max_t(unsigned int, |
| 165 | 2 * nr / BITS_PER_BYTE, L1_CACHE_BYTES)); |
| 166 | if (!data) |
| 167 | goto out_arr; |
| 168 | fdt->open_fds = (fd_set *)data; |
| 169 | data += nr / BITS_PER_BYTE; |
| 170 | fdt->close_on_exec = (fd_set *)data; |
| 171 | INIT_RCU_HEAD(&fdt->rcu); |
| 172 | fdt->next = NULL; |
| 173 | |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 174 | return fdt; |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 175 | |
| 176 | out_arr: |
| 177 | free_fdarr(fdt); |
| 178 | out_fdt: |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 179 | kfree(fdt); |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 180 | out: |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 181 | return NULL; |
| 182 | } |
| 183 | |
| 184 | /* |
Vadim Lobanov | 74d392a | 2006-09-29 02:01:43 -0700 | [diff] [blame] | 185 | * Expand the file descriptor table. |
| 186 | * This function will allocate a new fdtable and both fd array and fdset, of |
| 187 | * the given size. |
| 188 | * Return <0 error code on error; 1 on successful completion. |
| 189 | * The files->file_lock should be held on entry, and will be held on exit. |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 190 | */ |
| 191 | static int expand_fdtable(struct files_struct *files, int nr) |
| 192 | __releases(files->file_lock) |
| 193 | __acquires(files->file_lock) |
| 194 | { |
Vadim Lobanov | 74d392a | 2006-09-29 02:01:43 -0700 | [diff] [blame] | 195 | struct fdtable *new_fdt, *cur_fdt; |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 196 | |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 197 | spin_unlock(&files->file_lock); |
Vadim Lobanov | 74d392a | 2006-09-29 02:01:43 -0700 | [diff] [blame] | 198 | new_fdt = alloc_fdtable(nr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 199 | spin_lock(&files->file_lock); |
Vadim Lobanov | 74d392a | 2006-09-29 02:01:43 -0700 | [diff] [blame] | 200 | if (!new_fdt) |
| 201 | return -ENOMEM; |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 202 | /* |
Vadim Lobanov | 74d392a | 2006-09-29 02:01:43 -0700 | [diff] [blame] | 203 | * Check again since another task may have expanded the fd table while |
| 204 | * we dropped the lock |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 205 | */ |
Vadim Lobanov | 74d392a | 2006-09-29 02:01:43 -0700 | [diff] [blame] | 206 | cur_fdt = files_fdtable(files); |
Vadim Lobanov | bbea9f6 | 2006-12-10 02:21:12 -0800 | [diff] [blame] | 207 | if (nr >= cur_fdt->max_fds) { |
Vadim Lobanov | 74d392a | 2006-09-29 02:01:43 -0700 | [diff] [blame] | 208 | /* Continue as planned */ |
| 209 | copy_fdtable(new_fdt, cur_fdt); |
| 210 | rcu_assign_pointer(files->fdt, new_fdt); |
Vadim Lobanov | 4fd4581 | 2006-12-10 02:21:17 -0800 | [diff] [blame] | 211 | if (cur_fdt->max_fds > NR_OPEN_DEFAULT) |
Vadim Lobanov | 01b2d93 | 2006-12-22 01:10:43 -0800 | [diff] [blame] | 212 | free_fdtable(cur_fdt); |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 213 | } else { |
Vadim Lobanov | 74d392a | 2006-09-29 02:01:43 -0700 | [diff] [blame] | 214 | /* Somebody else expanded, so undo our attempt */ |
Vadim Lobanov | 5466b45 | 2006-12-10 02:21:22 -0800 | [diff] [blame] | 215 | free_fdarr(new_fdt); |
| 216 | free_fdset(new_fdt); |
| 217 | kfree(new_fdt); |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 218 | } |
Vadim Lobanov | 74d392a | 2006-09-29 02:01:43 -0700 | [diff] [blame] | 219 | return 1; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 220 | } |
| 221 | |
| 222 | /* |
| 223 | * Expand files. |
Vadim Lobanov | 74d392a | 2006-09-29 02:01:43 -0700 | [diff] [blame] | 224 | * This function will expand the file structures, if the requested size exceeds |
| 225 | * the current capacity and there is room for expansion. |
| 226 | * Return <0 error code on error; 0 when nothing done; 1 when files were |
| 227 | * expanded and execution may have blocked. |
| 228 | * The files->file_lock should be held on entry, and will be held on exit. |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 229 | */ |
| 230 | int expand_files(struct files_struct *files, int nr) |
| 231 | { |
Dipankar Sarma | badf166 | 2005-09-09 13:04:10 -0700 | [diff] [blame] | 232 | struct fdtable *fdt; |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 233 | |
Dipankar Sarma | badf166 | 2005-09-09 13:04:10 -0700 | [diff] [blame] | 234 | fdt = files_fdtable(files); |
Vadim Lobanov | 74d392a | 2006-09-29 02:01:43 -0700 | [diff] [blame] | 235 | /* Do we need to expand? */ |
Vadim Lobanov | bbea9f6 | 2006-12-10 02:21:12 -0800 | [diff] [blame] | 236 | if (nr < fdt->max_fds) |
Vadim Lobanov | 74d392a | 2006-09-29 02:01:43 -0700 | [diff] [blame] | 237 | return 0; |
| 238 | /* Can we expand? */ |
Eric Dumazet | 9cfe015 | 2008-02-06 01:37:16 -0800 | [diff] [blame] | 239 | if (nr >= sysctl_nr_open) |
Vadim Lobanov | 74d392a | 2006-09-29 02:01:43 -0700 | [diff] [blame] | 240 | return -EMFILE; |
| 241 | |
| 242 | /* All good, so we try */ |
| 243 | return expand_fdtable(files, nr); |
Linus Torvalds | 1da177e | 2005-04-16 15:20:36 -0700 | [diff] [blame] | 244 | } |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 245 | |
| 246 | static void __devinit fdtable_defer_list_init(int cpu) |
| 247 | { |
| 248 | struct fdtable_defer *fddef = &per_cpu(fdtable_defer_list, cpu); |
| 249 | spin_lock_init(&fddef->lock); |
David Howells | 65f27f3 | 2006-11-22 14:55:48 +0000 | [diff] [blame] | 250 | INIT_WORK(&fddef->wq, free_fdtable_work); |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 251 | fddef->next = NULL; |
| 252 | } |
| 253 | |
| 254 | void __init files_defer_init(void) |
| 255 | { |
| 256 | int i; |
KAMEZAWA Hiroyuki | 0a94502 | 2006-03-28 01:56:37 -0800 | [diff] [blame] | 257 | for_each_possible_cpu(i) |
Dipankar Sarma | ab2af1f | 2005-09-09 13:04:13 -0700 | [diff] [blame] | 258 | fdtable_defer_list_init(i); |
| 259 | } |