Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 1 | /* |
| 2 | * (C) 2001 Clemson University and The University of Chicago |
| 3 | * |
| 4 | * See COPYING in top-level directory. |
| 5 | */ |
| 6 | #include "protocol.h" |
Mike Marshall | 575e946 | 2015-12-04 12:56:14 -0500 | [diff] [blame] | 7 | #include "orangefs-kernel.h" |
| 8 | #include "orangefs-bufmap.h" |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 9 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 10 | DECLARE_WAIT_QUEUE_HEAD(orangefs_bufmap_init_waitq); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 11 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 12 | static struct orangefs_bufmap { |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 13 | atomic_t refcnt; |
| 14 | |
| 15 | int desc_size; |
| 16 | int desc_shift; |
| 17 | int desc_count; |
| 18 | int total_size; |
| 19 | int page_count; |
| 20 | |
| 21 | struct page **page_array; |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 22 | struct orangefs_bufmap_desc *desc_array; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 23 | |
| 24 | /* array to track usage of buffer descriptors */ |
| 25 | int *buffer_index_array; |
| 26 | spinlock_t buffer_index_lock; |
| 27 | |
| 28 | /* array to track usage of buffer descriptors for readdir */ |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 29 | int readdir_index_array[ORANGEFS_READDIR_DEFAULT_DESC_COUNT]; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 30 | spinlock_t readdir_index_lock; |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 31 | } *__orangefs_bufmap; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 32 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 33 | static DEFINE_SPINLOCK(orangefs_bufmap_lock); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 34 | |
| 35 | static void |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 36 | orangefs_bufmap_unmap(struct orangefs_bufmap *bufmap) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 37 | { |
| 38 | int i; |
| 39 | |
| 40 | for (i = 0; i < bufmap->page_count; i++) |
| 41 | page_cache_release(bufmap->page_array[i]); |
| 42 | } |
| 43 | |
| 44 | static void |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 45 | orangefs_bufmap_free(struct orangefs_bufmap *bufmap) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 46 | { |
| 47 | kfree(bufmap->page_array); |
| 48 | kfree(bufmap->desc_array); |
| 49 | kfree(bufmap->buffer_index_array); |
| 50 | kfree(bufmap); |
| 51 | } |
| 52 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 53 | struct orangefs_bufmap *orangefs_bufmap_ref(void) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 54 | { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 55 | struct orangefs_bufmap *bufmap = NULL; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 56 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 57 | spin_lock(&orangefs_bufmap_lock); |
| 58 | if (__orangefs_bufmap) { |
| 59 | bufmap = __orangefs_bufmap; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 60 | atomic_inc(&bufmap->refcnt); |
| 61 | } |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 62 | spin_unlock(&orangefs_bufmap_lock); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 63 | return bufmap; |
| 64 | } |
| 65 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 66 | void orangefs_bufmap_unref(struct orangefs_bufmap *bufmap) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 67 | { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 68 | if (atomic_dec_and_lock(&bufmap->refcnt, &orangefs_bufmap_lock)) { |
| 69 | __orangefs_bufmap = NULL; |
| 70 | spin_unlock(&orangefs_bufmap_lock); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 71 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 72 | orangefs_bufmap_unmap(bufmap); |
| 73 | orangefs_bufmap_free(bufmap); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 74 | } |
| 75 | } |
| 76 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 77 | inline int orangefs_bufmap_size_query(void) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 78 | { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 79 | struct orangefs_bufmap *bufmap = orangefs_bufmap_ref(); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 80 | int size = bufmap ? bufmap->desc_size : 0; |
| 81 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 82 | orangefs_bufmap_unref(bufmap); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 83 | return size; |
| 84 | } |
| 85 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 86 | inline int orangefs_bufmap_shift_query(void) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 87 | { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 88 | struct orangefs_bufmap *bufmap = orangefs_bufmap_ref(); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 89 | int shift = bufmap ? bufmap->desc_shift : 0; |
| 90 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 91 | orangefs_bufmap_unref(bufmap); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 92 | return shift; |
| 93 | } |
| 94 | |
| 95 | static DECLARE_WAIT_QUEUE_HEAD(bufmap_waitq); |
| 96 | static DECLARE_WAIT_QUEUE_HEAD(readdir_waitq); |
| 97 | |
| 98 | /* |
| 99 | * get_bufmap_init |
| 100 | * |
| 101 | * If bufmap_init is 1, then the shared memory system, including the |
| 102 | * buffer_index_array, is available. Otherwise, it is not. |
| 103 | * |
| 104 | * returns the value of bufmap_init |
| 105 | */ |
| 106 | int get_bufmap_init(void) |
| 107 | { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 108 | return __orangefs_bufmap ? 1 : 0; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 109 | } |
| 110 | |
| 111 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 112 | static struct orangefs_bufmap * |
| 113 | orangefs_bufmap_alloc(struct ORANGEFS_dev_map_desc *user_desc) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 114 | { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 115 | struct orangefs_bufmap *bufmap; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 116 | |
| 117 | bufmap = kzalloc(sizeof(*bufmap), GFP_KERNEL); |
| 118 | if (!bufmap) |
| 119 | goto out; |
| 120 | |
| 121 | atomic_set(&bufmap->refcnt, 1); |
| 122 | bufmap->total_size = user_desc->total_size; |
| 123 | bufmap->desc_count = user_desc->count; |
| 124 | bufmap->desc_size = user_desc->size; |
| 125 | bufmap->desc_shift = ilog2(bufmap->desc_size); |
| 126 | |
| 127 | spin_lock_init(&bufmap->buffer_index_lock); |
| 128 | bufmap->buffer_index_array = |
| 129 | kcalloc(bufmap->desc_count, sizeof(int), GFP_KERNEL); |
| 130 | if (!bufmap->buffer_index_array) { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 131 | gossip_err("orangefs: could not allocate %d buffer indices\n", |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 132 | bufmap->desc_count); |
| 133 | goto out_free_bufmap; |
| 134 | } |
| 135 | spin_lock_init(&bufmap->readdir_index_lock); |
| 136 | |
| 137 | bufmap->desc_array = |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 138 | kcalloc(bufmap->desc_count, sizeof(struct orangefs_bufmap_desc), |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 139 | GFP_KERNEL); |
| 140 | if (!bufmap->desc_array) { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 141 | gossip_err("orangefs: could not allocate %d descriptors\n", |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 142 | bufmap->desc_count); |
| 143 | goto out_free_index_array; |
| 144 | } |
| 145 | |
| 146 | bufmap->page_count = bufmap->total_size / PAGE_SIZE; |
| 147 | |
| 148 | /* allocate storage to track our page mappings */ |
| 149 | bufmap->page_array = |
| 150 | kcalloc(bufmap->page_count, sizeof(struct page *), GFP_KERNEL); |
| 151 | if (!bufmap->page_array) |
| 152 | goto out_free_desc_array; |
| 153 | |
| 154 | return bufmap; |
| 155 | |
| 156 | out_free_desc_array: |
| 157 | kfree(bufmap->desc_array); |
| 158 | out_free_index_array: |
| 159 | kfree(bufmap->buffer_index_array); |
| 160 | out_free_bufmap: |
| 161 | kfree(bufmap); |
| 162 | out: |
| 163 | return NULL; |
| 164 | } |
| 165 | |
| 166 | static int |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 167 | orangefs_bufmap_map(struct orangefs_bufmap *bufmap, |
| 168 | struct ORANGEFS_dev_map_desc *user_desc) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 169 | { |
| 170 | int pages_per_desc = bufmap->desc_size / PAGE_SIZE; |
| 171 | int offset = 0, ret, i; |
| 172 | |
| 173 | /* map the pages */ |
Al Viro | 16742f2 | 2015-10-08 20:10:00 -0400 | [diff] [blame] | 174 | ret = get_user_pages_fast((unsigned long)user_desc->ptr, |
| 175 | bufmap->page_count, 1, bufmap->page_array); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 176 | |
| 177 | if (ret < 0) |
| 178 | return ret; |
| 179 | |
| 180 | if (ret != bufmap->page_count) { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 181 | gossip_err("orangefs error: asked for %d pages, only got %d.\n", |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 182 | bufmap->page_count, ret); |
| 183 | |
| 184 | for (i = 0; i < ret; i++) { |
| 185 | SetPageError(bufmap->page_array[i]); |
| 186 | page_cache_release(bufmap->page_array[i]); |
| 187 | } |
| 188 | return -ENOMEM; |
| 189 | } |
| 190 | |
| 191 | /* |
| 192 | * ideally we want to get kernel space pointers for each page, but |
| 193 | * we can't kmap that many pages at once if highmem is being used. |
| 194 | * so instead, we just kmap/kunmap the page address each time the |
| 195 | * kaddr is needed. |
| 196 | */ |
| 197 | for (i = 0; i < bufmap->page_count; i++) |
| 198 | flush_dcache_page(bufmap->page_array[i]); |
| 199 | |
| 200 | /* build a list of available descriptors */ |
| 201 | for (offset = 0, i = 0; i < bufmap->desc_count; i++) { |
| 202 | bufmap->desc_array[i].page_array = &bufmap->page_array[offset]; |
| 203 | bufmap->desc_array[i].array_count = pages_per_desc; |
| 204 | bufmap->desc_array[i].uaddr = |
| 205 | (user_desc->ptr + (i * pages_per_desc * PAGE_SIZE)); |
| 206 | offset += pages_per_desc; |
| 207 | } |
| 208 | |
| 209 | return 0; |
| 210 | } |
| 211 | |
| 212 | /* |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 213 | * orangefs_bufmap_initialize() |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 214 | * |
| 215 | * initializes the mapped buffer interface |
| 216 | * |
| 217 | * returns 0 on success, -errno on failure |
| 218 | */ |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 219 | int orangefs_bufmap_initialize(struct ORANGEFS_dev_map_desc *user_desc) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 220 | { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 221 | struct orangefs_bufmap *bufmap; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 222 | int ret = -EINVAL; |
| 223 | |
| 224 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 225 | "orangefs_bufmap_initialize: called (ptr (" |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 226 | "%p) sz (%d) cnt(%d).\n", |
| 227 | user_desc->ptr, |
| 228 | user_desc->size, |
| 229 | user_desc->count); |
| 230 | |
| 231 | /* |
| 232 | * sanity check alignment and size of buffer that caller wants to |
| 233 | * work with |
| 234 | */ |
| 235 | if (PAGE_ALIGN((unsigned long)user_desc->ptr) != |
| 236 | (unsigned long)user_desc->ptr) { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 237 | gossip_err("orangefs error: memory alignment (front). %p\n", |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 238 | user_desc->ptr); |
| 239 | goto out; |
| 240 | } |
| 241 | |
| 242 | if (PAGE_ALIGN(((unsigned long)user_desc->ptr + user_desc->total_size)) |
| 243 | != (unsigned long)(user_desc->ptr + user_desc->total_size)) { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 244 | gossip_err("orangefs error: memory alignment (back).(%p + %d)\n", |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 245 | user_desc->ptr, |
| 246 | user_desc->total_size); |
| 247 | goto out; |
| 248 | } |
| 249 | |
| 250 | if (user_desc->total_size != (user_desc->size * user_desc->count)) { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 251 | gossip_err("orangefs error: user provided an oddly sized buffer: (%d, %d, %d)\n", |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 252 | user_desc->total_size, |
| 253 | user_desc->size, |
| 254 | user_desc->count); |
| 255 | goto out; |
| 256 | } |
| 257 | |
| 258 | if ((user_desc->size % PAGE_SIZE) != 0) { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 259 | gossip_err("orangefs error: bufmap size not page size divisible (%d).\n", |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 260 | user_desc->size); |
| 261 | goto out; |
| 262 | } |
| 263 | |
| 264 | ret = -ENOMEM; |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 265 | bufmap = orangefs_bufmap_alloc(user_desc); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 266 | if (!bufmap) |
| 267 | goto out; |
| 268 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 269 | ret = orangefs_bufmap_map(bufmap, user_desc); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 270 | if (ret) |
| 271 | goto out_free_bufmap; |
| 272 | |
| 273 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 274 | spin_lock(&orangefs_bufmap_lock); |
| 275 | if (__orangefs_bufmap) { |
| 276 | spin_unlock(&orangefs_bufmap_lock); |
| 277 | gossip_err("orangefs: error: bufmap already initialized.\n"); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 278 | ret = -EALREADY; |
| 279 | goto out_unmap_bufmap; |
| 280 | } |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 281 | __orangefs_bufmap = bufmap; |
| 282 | spin_unlock(&orangefs_bufmap_lock); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 283 | |
| 284 | /* |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 285 | * If there are operations in orangefs_bufmap_init_waitq, wake them up. |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 286 | * This scenario occurs when the client-core is restarted and I/O |
| 287 | * requests in the in-progress or waiting tables are restarted. I/O |
| 288 | * requests cannot be restarted until the shared memory system is |
| 289 | * completely re-initialized, so we put the I/O requests in this |
| 290 | * waitq until initialization has completed. NOTE: the I/O requests |
| 291 | * are also on a timer, so they don't wait forever just in case the |
| 292 | * client-core doesn't come back up. |
| 293 | */ |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 294 | wake_up_interruptible(&orangefs_bufmap_init_waitq); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 295 | |
| 296 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 297 | "orangefs_bufmap_initialize: exiting normally\n"); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 298 | return 0; |
| 299 | |
| 300 | out_unmap_bufmap: |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 301 | orangefs_bufmap_unmap(bufmap); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 302 | out_free_bufmap: |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 303 | orangefs_bufmap_free(bufmap); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 304 | out: |
| 305 | return ret; |
| 306 | } |
| 307 | |
| 308 | /* |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 309 | * orangefs_bufmap_finalize() |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 310 | * |
| 311 | * shuts down the mapped buffer interface and releases any resources |
| 312 | * associated with it |
| 313 | * |
| 314 | * no return value |
| 315 | */ |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 316 | void orangefs_bufmap_finalize(void) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 317 | { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 318 | gossip_debug(GOSSIP_BUFMAP_DEBUG, "orangefs_bufmap_finalize: called\n"); |
| 319 | BUG_ON(!__orangefs_bufmap); |
| 320 | orangefs_bufmap_unref(__orangefs_bufmap); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 321 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 322 | "orangefs_bufmap_finalize: exiting normally\n"); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 323 | } |
| 324 | |
| 325 | struct slot_args { |
| 326 | int slot_count; |
| 327 | int *slot_array; |
| 328 | spinlock_t *slot_lock; |
| 329 | wait_queue_head_t *slot_wq; |
| 330 | }; |
| 331 | |
| 332 | static int wait_for_a_slot(struct slot_args *slargs, int *buffer_index) |
| 333 | { |
| 334 | int ret = -1; |
| 335 | int i = 0; |
| 336 | DECLARE_WAITQUEUE(my_wait, current); |
| 337 | |
| 338 | |
| 339 | add_wait_queue_exclusive(slargs->slot_wq, &my_wait); |
| 340 | |
| 341 | while (1) { |
| 342 | set_current_state(TASK_INTERRUPTIBLE); |
| 343 | |
| 344 | /* |
| 345 | * check for available desc, slot_lock is the appropriate |
| 346 | * index_lock |
| 347 | */ |
| 348 | spin_lock(slargs->slot_lock); |
| 349 | for (i = 0; i < slargs->slot_count; i++) |
| 350 | if (slargs->slot_array[i] == 0) { |
| 351 | slargs->slot_array[i] = 1; |
| 352 | *buffer_index = i; |
| 353 | ret = 0; |
| 354 | break; |
| 355 | } |
| 356 | spin_unlock(slargs->slot_lock); |
| 357 | |
| 358 | /* if we acquired a buffer, then break out of while */ |
| 359 | if (ret == 0) |
| 360 | break; |
| 361 | |
| 362 | if (!signal_pending(current)) { |
| 363 | int timeout = |
| 364 | MSECS_TO_JIFFIES(1000 * slot_timeout_secs); |
| 365 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 366 | "[BUFMAP]: waiting %d " |
| 367 | "seconds for a slot\n", |
| 368 | slot_timeout_secs); |
| 369 | if (!schedule_timeout(timeout)) { |
| 370 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 371 | "*** wait_for_a_slot timed out\n"); |
| 372 | ret = -ETIMEDOUT; |
| 373 | break; |
| 374 | } |
| 375 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 376 | "[BUFMAP]: woken up by a slot becoming available.\n"); |
| 377 | continue; |
| 378 | } |
| 379 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 380 | gossip_debug(GOSSIP_BUFMAP_DEBUG, "orangefs: %s interrupted.\n", |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 381 | __func__); |
| 382 | ret = -EINTR; |
| 383 | break; |
| 384 | } |
| 385 | |
| 386 | set_current_state(TASK_RUNNING); |
| 387 | remove_wait_queue(slargs->slot_wq, &my_wait); |
| 388 | return ret; |
| 389 | } |
| 390 | |
| 391 | static void put_back_slot(struct slot_args *slargs, int buffer_index) |
| 392 | { |
| 393 | /* slot_lock is the appropriate index_lock */ |
| 394 | spin_lock(slargs->slot_lock); |
| 395 | if (buffer_index < 0 || buffer_index >= slargs->slot_count) { |
| 396 | spin_unlock(slargs->slot_lock); |
| 397 | return; |
| 398 | } |
| 399 | |
| 400 | /* put the desc back on the queue */ |
| 401 | slargs->slot_array[buffer_index] = 0; |
| 402 | spin_unlock(slargs->slot_lock); |
| 403 | |
| 404 | /* wake up anyone who may be sleeping on the queue */ |
| 405 | wake_up_interruptible(slargs->slot_wq); |
| 406 | } |
| 407 | |
| 408 | /* |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 409 | * orangefs_bufmap_get() |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 410 | * |
| 411 | * gets a free mapped buffer descriptor, will sleep until one becomes |
| 412 | * available if necessary |
| 413 | * |
| 414 | * returns 0 on success, -errno on failure |
| 415 | */ |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 416 | int orangefs_bufmap_get(struct orangefs_bufmap **mapp, int *buffer_index) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 417 | { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 418 | struct orangefs_bufmap *bufmap = orangefs_bufmap_ref(); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 419 | struct slot_args slargs; |
| 420 | int ret; |
| 421 | |
| 422 | if (!bufmap) { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 423 | gossip_err("orangefs: please confirm that pvfs2-client daemon is running.\n"); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 424 | return -EIO; |
| 425 | } |
| 426 | |
| 427 | slargs.slot_count = bufmap->desc_count; |
| 428 | slargs.slot_array = bufmap->buffer_index_array; |
| 429 | slargs.slot_lock = &bufmap->buffer_index_lock; |
| 430 | slargs.slot_wq = &bufmap_waitq; |
| 431 | ret = wait_for_a_slot(&slargs, buffer_index); |
| 432 | if (ret) |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 433 | orangefs_bufmap_unref(bufmap); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 434 | *mapp = bufmap; |
| 435 | return ret; |
| 436 | } |
| 437 | |
| 438 | /* |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 439 | * orangefs_bufmap_put() |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 440 | * |
| 441 | * returns a mapped buffer descriptor to the collection |
| 442 | * |
| 443 | * no return value |
| 444 | */ |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 445 | void orangefs_bufmap_put(struct orangefs_bufmap *bufmap, int buffer_index) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 446 | { |
| 447 | struct slot_args slargs; |
| 448 | |
| 449 | slargs.slot_count = bufmap->desc_count; |
| 450 | slargs.slot_array = bufmap->buffer_index_array; |
| 451 | slargs.slot_lock = &bufmap->buffer_index_lock; |
| 452 | slargs.slot_wq = &bufmap_waitq; |
| 453 | put_back_slot(&slargs, buffer_index); |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 454 | orangefs_bufmap_unref(bufmap); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 455 | } |
| 456 | |
| 457 | /* |
| 458 | * readdir_index_get() |
| 459 | * |
| 460 | * gets a free descriptor, will sleep until one becomes |
| 461 | * available if necessary. |
| 462 | * Although the readdir buffers are not mapped into kernel space |
| 463 | * we could do that at a later point of time. Regardless, these |
| 464 | * indices are used by the client-core. |
| 465 | * |
| 466 | * returns 0 on success, -errno on failure |
| 467 | */ |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 468 | int readdir_index_get(struct orangefs_bufmap **mapp, int *buffer_index) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 469 | { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 470 | struct orangefs_bufmap *bufmap = orangefs_bufmap_ref(); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 471 | struct slot_args slargs; |
| 472 | int ret; |
| 473 | |
| 474 | if (!bufmap) { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 475 | gossip_err("orangefs: please confirm that pvfs2-client daemon is running.\n"); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 476 | return -EIO; |
| 477 | } |
| 478 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 479 | slargs.slot_count = ORANGEFS_READDIR_DEFAULT_DESC_COUNT; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 480 | slargs.slot_array = bufmap->readdir_index_array; |
| 481 | slargs.slot_lock = &bufmap->readdir_index_lock; |
| 482 | slargs.slot_wq = &readdir_waitq; |
| 483 | ret = wait_for_a_slot(&slargs, buffer_index); |
| 484 | if (ret) |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 485 | orangefs_bufmap_unref(bufmap); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 486 | *mapp = bufmap; |
| 487 | return ret; |
| 488 | } |
| 489 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 490 | void readdir_index_put(struct orangefs_bufmap *bufmap, int buffer_index) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 491 | { |
| 492 | struct slot_args slargs; |
| 493 | |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 494 | slargs.slot_count = ORANGEFS_READDIR_DEFAULT_DESC_COUNT; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 495 | slargs.slot_array = bufmap->readdir_index_array; |
| 496 | slargs.slot_lock = &bufmap->readdir_index_lock; |
| 497 | slargs.slot_wq = &readdir_waitq; |
| 498 | put_back_slot(&slargs, buffer_index); |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 499 | orangefs_bufmap_unref(bufmap); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 500 | } |
| 501 | |
Mike Marshall | b5e376e | 2015-12-11 10:50:42 -0500 | [diff] [blame^] | 502 | /* |
| 503 | * we've been handed an iovec, we need to copy it to |
| 504 | * the shared memory descriptor at "buffer_index". |
| 505 | */ |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 506 | int orangefs_bufmap_copy_from_iovec(struct orangefs_bufmap *bufmap, |
Mike Marshall | 5480494 | 2015-10-05 13:44:24 -0400 | [diff] [blame] | 507 | struct iov_iter *iter, |
| 508 | int buffer_index, |
| 509 | size_t size) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 510 | { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 511 | struct orangefs_bufmap_desc *to = &bufmap->desc_array[buffer_index]; |
Mike Marshall | 4d1c440 | 2015-09-04 10:31:16 -0400 | [diff] [blame] | 512 | int i; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 513 | |
| 514 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
Al Viro | 34204fd | 2015-10-08 17:47:44 -0400 | [diff] [blame] | 515 | "%s: buffer_index:%d: size:%zu:\n", |
Mike Marshall | 4d1c440 | 2015-09-04 10:31:16 -0400 | [diff] [blame] | 516 | __func__, buffer_index, size); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 517 | |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 518 | |
Mike Marshall | 4d1c440 | 2015-09-04 10:31:16 -0400 | [diff] [blame] | 519 | for (i = 0; size; i++) { |
Al Viro | 34204fd | 2015-10-08 17:47:44 -0400 | [diff] [blame] | 520 | struct page *page = to->page_array[i]; |
| 521 | size_t n = size; |
| 522 | if (n > PAGE_SIZE) |
| 523 | n = PAGE_SIZE; |
| 524 | n = copy_page_from_iter(page, 0, n, iter); |
| 525 | if (!n) |
| 526 | return -EFAULT; |
| 527 | size -= n; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 528 | } |
Al Viro | 34204fd | 2015-10-08 17:47:44 -0400 | [diff] [blame] | 529 | return 0; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 530 | |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 531 | } |
| 532 | |
| 533 | /* |
Mike Marshall | b5e376e | 2015-12-11 10:50:42 -0500 | [diff] [blame^] | 534 | * we've been handed an iovec, we need to fill it from |
| 535 | * the shared memory descriptor at "buffer_index". |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 536 | */ |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 537 | int orangefs_bufmap_copy_to_iovec(struct orangefs_bufmap *bufmap, |
Mike Marshall | 4d1c440 | 2015-09-04 10:31:16 -0400 | [diff] [blame] | 538 | struct iov_iter *iter, |
Al Viro | 5c27822 | 2015-10-08 17:43:58 -0400 | [diff] [blame] | 539 | int buffer_index, |
| 540 | size_t size) |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 541 | { |
Yi Liu | 8bb8aef | 2015-11-24 15:12:14 -0500 | [diff] [blame] | 542 | struct orangefs_bufmap_desc *from = &bufmap->desc_array[buffer_index]; |
Mike Marshall | 4d1c440 | 2015-09-04 10:31:16 -0400 | [diff] [blame] | 543 | int i; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 544 | |
| 545 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
Al Viro | 5c27822 | 2015-10-08 17:43:58 -0400 | [diff] [blame] | 546 | "%s: buffer_index:%d: size:%zu:\n", |
| 547 | __func__, buffer_index, size); |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 548 | |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 549 | |
Al Viro | 5c27822 | 2015-10-08 17:43:58 -0400 | [diff] [blame] | 550 | for (i = 0; size; i++) { |
| 551 | struct page *page = from->page_array[i]; |
| 552 | size_t n = size; |
| 553 | if (n > PAGE_SIZE) |
| 554 | n = PAGE_SIZE; |
| 555 | n = copy_page_to_iter(page, 0, n, iter); |
| 556 | if (!n) |
| 557 | return -EFAULT; |
| 558 | size -= n; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 559 | } |
Al Viro | 5c27822 | 2015-10-08 17:43:58 -0400 | [diff] [blame] | 560 | return 0; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 561 | } |