Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 1 | /* |
| 2 | * (C) 2001 Clemson University and The University of Chicago |
| 3 | * |
| 4 | * See COPYING in top-level directory. |
| 5 | */ |
| 6 | #include "protocol.h" |
| 7 | #include "pvfs2-kernel.h" |
| 8 | #include "pvfs2-bufmap.h" |
| 9 | |
| 10 | DECLARE_WAIT_QUEUE_HEAD(pvfs2_bufmap_init_waitq); |
| 11 | |
Mike Marshall | 84d0215 | 2015-07-28 13:27:51 -0400 | [diff] [blame] | 12 | static struct pvfs2_bufmap { |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 13 | atomic_t refcnt; |
| 14 | |
| 15 | int desc_size; |
| 16 | int desc_shift; |
| 17 | int desc_count; |
| 18 | int total_size; |
| 19 | int page_count; |
| 20 | |
| 21 | struct page **page_array; |
| 22 | struct pvfs_bufmap_desc *desc_array; |
| 23 | |
| 24 | /* array to track usage of buffer descriptors */ |
| 25 | int *buffer_index_array; |
| 26 | spinlock_t buffer_index_lock; |
| 27 | |
| 28 | /* array to track usage of buffer descriptors for readdir */ |
| 29 | int readdir_index_array[PVFS2_READDIR_DEFAULT_DESC_COUNT]; |
| 30 | spinlock_t readdir_index_lock; |
| 31 | } *__pvfs2_bufmap; |
| 32 | |
| 33 | static DEFINE_SPINLOCK(pvfs2_bufmap_lock); |
| 34 | |
| 35 | static void |
| 36 | pvfs2_bufmap_unmap(struct pvfs2_bufmap *bufmap) |
| 37 | { |
| 38 | int i; |
| 39 | |
| 40 | for (i = 0; i < bufmap->page_count; i++) |
| 41 | page_cache_release(bufmap->page_array[i]); |
| 42 | } |
| 43 | |
| 44 | static void |
| 45 | pvfs2_bufmap_free(struct pvfs2_bufmap *bufmap) |
| 46 | { |
| 47 | kfree(bufmap->page_array); |
| 48 | kfree(bufmap->desc_array); |
| 49 | kfree(bufmap->buffer_index_array); |
| 50 | kfree(bufmap); |
| 51 | } |
| 52 | |
| 53 | struct pvfs2_bufmap *pvfs2_bufmap_ref(void) |
| 54 | { |
| 55 | struct pvfs2_bufmap *bufmap = NULL; |
| 56 | |
| 57 | spin_lock(&pvfs2_bufmap_lock); |
| 58 | if (__pvfs2_bufmap) { |
| 59 | bufmap = __pvfs2_bufmap; |
| 60 | atomic_inc(&bufmap->refcnt); |
| 61 | } |
| 62 | spin_unlock(&pvfs2_bufmap_lock); |
| 63 | return bufmap; |
| 64 | } |
| 65 | |
| 66 | void pvfs2_bufmap_unref(struct pvfs2_bufmap *bufmap) |
| 67 | { |
| 68 | if (atomic_dec_and_lock(&bufmap->refcnt, &pvfs2_bufmap_lock)) { |
| 69 | __pvfs2_bufmap = NULL; |
| 70 | spin_unlock(&pvfs2_bufmap_lock); |
| 71 | |
| 72 | pvfs2_bufmap_unmap(bufmap); |
| 73 | pvfs2_bufmap_free(bufmap); |
| 74 | } |
| 75 | } |
| 76 | |
| 77 | inline int pvfs_bufmap_size_query(void) |
| 78 | { |
| 79 | struct pvfs2_bufmap *bufmap = pvfs2_bufmap_ref(); |
| 80 | int size = bufmap ? bufmap->desc_size : 0; |
| 81 | |
| 82 | pvfs2_bufmap_unref(bufmap); |
| 83 | return size; |
| 84 | } |
| 85 | |
| 86 | inline int pvfs_bufmap_shift_query(void) |
| 87 | { |
| 88 | struct pvfs2_bufmap *bufmap = pvfs2_bufmap_ref(); |
| 89 | int shift = bufmap ? bufmap->desc_shift : 0; |
| 90 | |
| 91 | pvfs2_bufmap_unref(bufmap); |
| 92 | return shift; |
| 93 | } |
| 94 | |
| 95 | static DECLARE_WAIT_QUEUE_HEAD(bufmap_waitq); |
| 96 | static DECLARE_WAIT_QUEUE_HEAD(readdir_waitq); |
| 97 | |
| 98 | /* |
| 99 | * get_bufmap_init |
| 100 | * |
| 101 | * If bufmap_init is 1, then the shared memory system, including the |
| 102 | * buffer_index_array, is available. Otherwise, it is not. |
| 103 | * |
| 104 | * returns the value of bufmap_init |
| 105 | */ |
| 106 | int get_bufmap_init(void) |
| 107 | { |
| 108 | return __pvfs2_bufmap ? 1 : 0; |
| 109 | } |
| 110 | |
| 111 | |
| 112 | static struct pvfs2_bufmap * |
| 113 | pvfs2_bufmap_alloc(struct PVFS_dev_map_desc *user_desc) |
| 114 | { |
| 115 | struct pvfs2_bufmap *bufmap; |
| 116 | |
| 117 | bufmap = kzalloc(sizeof(*bufmap), GFP_KERNEL); |
| 118 | if (!bufmap) |
| 119 | goto out; |
| 120 | |
| 121 | atomic_set(&bufmap->refcnt, 1); |
| 122 | bufmap->total_size = user_desc->total_size; |
| 123 | bufmap->desc_count = user_desc->count; |
| 124 | bufmap->desc_size = user_desc->size; |
| 125 | bufmap->desc_shift = ilog2(bufmap->desc_size); |
| 126 | |
| 127 | spin_lock_init(&bufmap->buffer_index_lock); |
| 128 | bufmap->buffer_index_array = |
| 129 | kcalloc(bufmap->desc_count, sizeof(int), GFP_KERNEL); |
| 130 | if (!bufmap->buffer_index_array) { |
| 131 | gossip_err("pvfs2: could not allocate %d buffer indices\n", |
| 132 | bufmap->desc_count); |
| 133 | goto out_free_bufmap; |
| 134 | } |
| 135 | spin_lock_init(&bufmap->readdir_index_lock); |
| 136 | |
| 137 | bufmap->desc_array = |
| 138 | kcalloc(bufmap->desc_count, sizeof(struct pvfs_bufmap_desc), |
| 139 | GFP_KERNEL); |
| 140 | if (!bufmap->desc_array) { |
| 141 | gossip_err("pvfs2: could not allocate %d descriptors\n", |
| 142 | bufmap->desc_count); |
| 143 | goto out_free_index_array; |
| 144 | } |
| 145 | |
| 146 | bufmap->page_count = bufmap->total_size / PAGE_SIZE; |
| 147 | |
| 148 | /* allocate storage to track our page mappings */ |
| 149 | bufmap->page_array = |
| 150 | kcalloc(bufmap->page_count, sizeof(struct page *), GFP_KERNEL); |
| 151 | if (!bufmap->page_array) |
| 152 | goto out_free_desc_array; |
| 153 | |
| 154 | return bufmap; |
| 155 | |
| 156 | out_free_desc_array: |
| 157 | kfree(bufmap->desc_array); |
| 158 | out_free_index_array: |
| 159 | kfree(bufmap->buffer_index_array); |
| 160 | out_free_bufmap: |
| 161 | kfree(bufmap); |
| 162 | out: |
| 163 | return NULL; |
| 164 | } |
| 165 | |
| 166 | static int |
| 167 | pvfs2_bufmap_map(struct pvfs2_bufmap *bufmap, |
| 168 | struct PVFS_dev_map_desc *user_desc) |
| 169 | { |
| 170 | int pages_per_desc = bufmap->desc_size / PAGE_SIZE; |
| 171 | int offset = 0, ret, i; |
| 172 | |
| 173 | /* map the pages */ |
| 174 | down_write(¤t->mm->mmap_sem); |
| 175 | ret = get_user_pages(current, |
| 176 | current->mm, |
| 177 | (unsigned long)user_desc->ptr, |
| 178 | bufmap->page_count, |
| 179 | 1, |
| 180 | 0, |
| 181 | bufmap->page_array, |
| 182 | NULL); |
| 183 | up_write(¤t->mm->mmap_sem); |
| 184 | |
| 185 | if (ret < 0) |
| 186 | return ret; |
| 187 | |
| 188 | if (ret != bufmap->page_count) { |
| 189 | gossip_err("pvfs2 error: asked for %d pages, only got %d.\n", |
| 190 | bufmap->page_count, ret); |
| 191 | |
| 192 | for (i = 0; i < ret; i++) { |
| 193 | SetPageError(bufmap->page_array[i]); |
| 194 | page_cache_release(bufmap->page_array[i]); |
| 195 | } |
| 196 | return -ENOMEM; |
| 197 | } |
| 198 | |
| 199 | /* |
| 200 | * ideally we want to get kernel space pointers for each page, but |
| 201 | * we can't kmap that many pages at once if highmem is being used. |
| 202 | * so instead, we just kmap/kunmap the page address each time the |
| 203 | * kaddr is needed. |
| 204 | */ |
| 205 | for (i = 0; i < bufmap->page_count; i++) |
| 206 | flush_dcache_page(bufmap->page_array[i]); |
| 207 | |
| 208 | /* build a list of available descriptors */ |
| 209 | for (offset = 0, i = 0; i < bufmap->desc_count; i++) { |
| 210 | bufmap->desc_array[i].page_array = &bufmap->page_array[offset]; |
| 211 | bufmap->desc_array[i].array_count = pages_per_desc; |
| 212 | bufmap->desc_array[i].uaddr = |
| 213 | (user_desc->ptr + (i * pages_per_desc * PAGE_SIZE)); |
| 214 | offset += pages_per_desc; |
| 215 | } |
| 216 | |
| 217 | return 0; |
| 218 | } |
| 219 | |
| 220 | /* |
| 221 | * pvfs_bufmap_initialize() |
| 222 | * |
| 223 | * initializes the mapped buffer interface |
| 224 | * |
| 225 | * returns 0 on success, -errno on failure |
| 226 | */ |
| 227 | int pvfs_bufmap_initialize(struct PVFS_dev_map_desc *user_desc) |
| 228 | { |
| 229 | struct pvfs2_bufmap *bufmap; |
| 230 | int ret = -EINVAL; |
| 231 | |
| 232 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 233 | "pvfs_bufmap_initialize: called (ptr (" |
| 234 | "%p) sz (%d) cnt(%d).\n", |
| 235 | user_desc->ptr, |
| 236 | user_desc->size, |
| 237 | user_desc->count); |
| 238 | |
| 239 | /* |
| 240 | * sanity check alignment and size of buffer that caller wants to |
| 241 | * work with |
| 242 | */ |
| 243 | if (PAGE_ALIGN((unsigned long)user_desc->ptr) != |
| 244 | (unsigned long)user_desc->ptr) { |
| 245 | gossip_err("pvfs2 error: memory alignment (front). %p\n", |
| 246 | user_desc->ptr); |
| 247 | goto out; |
| 248 | } |
| 249 | |
| 250 | if (PAGE_ALIGN(((unsigned long)user_desc->ptr + user_desc->total_size)) |
| 251 | != (unsigned long)(user_desc->ptr + user_desc->total_size)) { |
| 252 | gossip_err("pvfs2 error: memory alignment (back).(%p + %d)\n", |
| 253 | user_desc->ptr, |
| 254 | user_desc->total_size); |
| 255 | goto out; |
| 256 | } |
| 257 | |
| 258 | if (user_desc->total_size != (user_desc->size * user_desc->count)) { |
| 259 | gossip_err("pvfs2 error: user provided an oddly sized buffer: (%d, %d, %d)\n", |
| 260 | user_desc->total_size, |
| 261 | user_desc->size, |
| 262 | user_desc->count); |
| 263 | goto out; |
| 264 | } |
| 265 | |
| 266 | if ((user_desc->size % PAGE_SIZE) != 0) { |
| 267 | gossip_err("pvfs2 error: bufmap size not page size divisible (%d).\n", |
| 268 | user_desc->size); |
| 269 | goto out; |
| 270 | } |
| 271 | |
| 272 | ret = -ENOMEM; |
| 273 | bufmap = pvfs2_bufmap_alloc(user_desc); |
| 274 | if (!bufmap) |
| 275 | goto out; |
| 276 | |
| 277 | ret = pvfs2_bufmap_map(bufmap, user_desc); |
| 278 | if (ret) |
| 279 | goto out_free_bufmap; |
| 280 | |
| 281 | |
| 282 | spin_lock(&pvfs2_bufmap_lock); |
| 283 | if (__pvfs2_bufmap) { |
| 284 | spin_unlock(&pvfs2_bufmap_lock); |
| 285 | gossip_err("pvfs2: error: bufmap already initialized.\n"); |
| 286 | ret = -EALREADY; |
| 287 | goto out_unmap_bufmap; |
| 288 | } |
| 289 | __pvfs2_bufmap = bufmap; |
| 290 | spin_unlock(&pvfs2_bufmap_lock); |
| 291 | |
| 292 | /* |
| 293 | * If there are operations in pvfs2_bufmap_init_waitq, wake them up. |
| 294 | * This scenario occurs when the client-core is restarted and I/O |
| 295 | * requests in the in-progress or waiting tables are restarted. I/O |
| 296 | * requests cannot be restarted until the shared memory system is |
| 297 | * completely re-initialized, so we put the I/O requests in this |
| 298 | * waitq until initialization has completed. NOTE: the I/O requests |
| 299 | * are also on a timer, so they don't wait forever just in case the |
| 300 | * client-core doesn't come back up. |
| 301 | */ |
| 302 | wake_up_interruptible(&pvfs2_bufmap_init_waitq); |
| 303 | |
| 304 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 305 | "pvfs_bufmap_initialize: exiting normally\n"); |
| 306 | return 0; |
| 307 | |
| 308 | out_unmap_bufmap: |
| 309 | pvfs2_bufmap_unmap(bufmap); |
| 310 | out_free_bufmap: |
| 311 | pvfs2_bufmap_free(bufmap); |
| 312 | out: |
| 313 | return ret; |
| 314 | } |
| 315 | |
| 316 | /* |
| 317 | * pvfs_bufmap_finalize() |
| 318 | * |
| 319 | * shuts down the mapped buffer interface and releases any resources |
| 320 | * associated with it |
| 321 | * |
| 322 | * no return value |
| 323 | */ |
| 324 | void pvfs_bufmap_finalize(void) |
| 325 | { |
| 326 | gossip_debug(GOSSIP_BUFMAP_DEBUG, "pvfs2_bufmap_finalize: called\n"); |
| 327 | BUG_ON(!__pvfs2_bufmap); |
| 328 | pvfs2_bufmap_unref(__pvfs2_bufmap); |
| 329 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 330 | "pvfs2_bufmap_finalize: exiting normally\n"); |
| 331 | } |
| 332 | |
| 333 | struct slot_args { |
| 334 | int slot_count; |
| 335 | int *slot_array; |
| 336 | spinlock_t *slot_lock; |
| 337 | wait_queue_head_t *slot_wq; |
| 338 | }; |
| 339 | |
| 340 | static int wait_for_a_slot(struct slot_args *slargs, int *buffer_index) |
| 341 | { |
| 342 | int ret = -1; |
| 343 | int i = 0; |
| 344 | DECLARE_WAITQUEUE(my_wait, current); |
| 345 | |
| 346 | |
| 347 | add_wait_queue_exclusive(slargs->slot_wq, &my_wait); |
| 348 | |
| 349 | while (1) { |
| 350 | set_current_state(TASK_INTERRUPTIBLE); |
| 351 | |
| 352 | /* |
| 353 | * check for available desc, slot_lock is the appropriate |
| 354 | * index_lock |
| 355 | */ |
| 356 | spin_lock(slargs->slot_lock); |
| 357 | for (i = 0; i < slargs->slot_count; i++) |
| 358 | if (slargs->slot_array[i] == 0) { |
| 359 | slargs->slot_array[i] = 1; |
| 360 | *buffer_index = i; |
| 361 | ret = 0; |
| 362 | break; |
| 363 | } |
| 364 | spin_unlock(slargs->slot_lock); |
| 365 | |
| 366 | /* if we acquired a buffer, then break out of while */ |
| 367 | if (ret == 0) |
| 368 | break; |
| 369 | |
| 370 | if (!signal_pending(current)) { |
| 371 | int timeout = |
| 372 | MSECS_TO_JIFFIES(1000 * slot_timeout_secs); |
| 373 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 374 | "[BUFMAP]: waiting %d " |
| 375 | "seconds for a slot\n", |
| 376 | slot_timeout_secs); |
| 377 | if (!schedule_timeout(timeout)) { |
| 378 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 379 | "*** wait_for_a_slot timed out\n"); |
| 380 | ret = -ETIMEDOUT; |
| 381 | break; |
| 382 | } |
| 383 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 384 | "[BUFMAP]: woken up by a slot becoming available.\n"); |
| 385 | continue; |
| 386 | } |
| 387 | |
| 388 | gossip_debug(GOSSIP_BUFMAP_DEBUG, "pvfs2: %s interrupted.\n", |
| 389 | __func__); |
| 390 | ret = -EINTR; |
| 391 | break; |
| 392 | } |
| 393 | |
| 394 | set_current_state(TASK_RUNNING); |
| 395 | remove_wait_queue(slargs->slot_wq, &my_wait); |
| 396 | return ret; |
| 397 | } |
| 398 | |
| 399 | static void put_back_slot(struct slot_args *slargs, int buffer_index) |
| 400 | { |
| 401 | /* slot_lock is the appropriate index_lock */ |
| 402 | spin_lock(slargs->slot_lock); |
| 403 | if (buffer_index < 0 || buffer_index >= slargs->slot_count) { |
| 404 | spin_unlock(slargs->slot_lock); |
| 405 | return; |
| 406 | } |
| 407 | |
| 408 | /* put the desc back on the queue */ |
| 409 | slargs->slot_array[buffer_index] = 0; |
| 410 | spin_unlock(slargs->slot_lock); |
| 411 | |
| 412 | /* wake up anyone who may be sleeping on the queue */ |
| 413 | wake_up_interruptible(slargs->slot_wq); |
| 414 | } |
| 415 | |
| 416 | /* |
| 417 | * pvfs_bufmap_get() |
| 418 | * |
| 419 | * gets a free mapped buffer descriptor, will sleep until one becomes |
| 420 | * available if necessary |
| 421 | * |
| 422 | * returns 0 on success, -errno on failure |
| 423 | */ |
| 424 | int pvfs_bufmap_get(struct pvfs2_bufmap **mapp, int *buffer_index) |
| 425 | { |
| 426 | struct pvfs2_bufmap *bufmap = pvfs2_bufmap_ref(); |
| 427 | struct slot_args slargs; |
| 428 | int ret; |
| 429 | |
| 430 | if (!bufmap) { |
| 431 | gossip_err("pvfs2: please confirm that pvfs2-client daemon is running.\n"); |
| 432 | return -EIO; |
| 433 | } |
| 434 | |
| 435 | slargs.slot_count = bufmap->desc_count; |
| 436 | slargs.slot_array = bufmap->buffer_index_array; |
| 437 | slargs.slot_lock = &bufmap->buffer_index_lock; |
| 438 | slargs.slot_wq = &bufmap_waitq; |
| 439 | ret = wait_for_a_slot(&slargs, buffer_index); |
| 440 | if (ret) |
| 441 | pvfs2_bufmap_unref(bufmap); |
| 442 | *mapp = bufmap; |
| 443 | return ret; |
| 444 | } |
| 445 | |
| 446 | /* |
| 447 | * pvfs_bufmap_put() |
| 448 | * |
| 449 | * returns a mapped buffer descriptor to the collection |
| 450 | * |
| 451 | * no return value |
| 452 | */ |
| 453 | void pvfs_bufmap_put(struct pvfs2_bufmap *bufmap, int buffer_index) |
| 454 | { |
| 455 | struct slot_args slargs; |
| 456 | |
| 457 | slargs.slot_count = bufmap->desc_count; |
| 458 | slargs.slot_array = bufmap->buffer_index_array; |
| 459 | slargs.slot_lock = &bufmap->buffer_index_lock; |
| 460 | slargs.slot_wq = &bufmap_waitq; |
| 461 | put_back_slot(&slargs, buffer_index); |
| 462 | pvfs2_bufmap_unref(bufmap); |
| 463 | } |
| 464 | |
| 465 | /* |
| 466 | * readdir_index_get() |
| 467 | * |
| 468 | * gets a free descriptor, will sleep until one becomes |
| 469 | * available if necessary. |
| 470 | * Although the readdir buffers are not mapped into kernel space |
| 471 | * we could do that at a later point of time. Regardless, these |
| 472 | * indices are used by the client-core. |
| 473 | * |
| 474 | * returns 0 on success, -errno on failure |
| 475 | */ |
| 476 | int readdir_index_get(struct pvfs2_bufmap **mapp, int *buffer_index) |
| 477 | { |
| 478 | struct pvfs2_bufmap *bufmap = pvfs2_bufmap_ref(); |
| 479 | struct slot_args slargs; |
| 480 | int ret; |
| 481 | |
| 482 | if (!bufmap) { |
| 483 | gossip_err("pvfs2: please confirm that pvfs2-client daemon is running.\n"); |
| 484 | return -EIO; |
| 485 | } |
| 486 | |
| 487 | slargs.slot_count = PVFS2_READDIR_DEFAULT_DESC_COUNT; |
| 488 | slargs.slot_array = bufmap->readdir_index_array; |
| 489 | slargs.slot_lock = &bufmap->readdir_index_lock; |
| 490 | slargs.slot_wq = &readdir_waitq; |
| 491 | ret = wait_for_a_slot(&slargs, buffer_index); |
| 492 | if (ret) |
| 493 | pvfs2_bufmap_unref(bufmap); |
| 494 | *mapp = bufmap; |
| 495 | return ret; |
| 496 | } |
| 497 | |
| 498 | void readdir_index_put(struct pvfs2_bufmap *bufmap, int buffer_index) |
| 499 | { |
| 500 | struct slot_args slargs; |
| 501 | |
| 502 | slargs.slot_count = PVFS2_READDIR_DEFAULT_DESC_COUNT; |
| 503 | slargs.slot_array = bufmap->readdir_index_array; |
| 504 | slargs.slot_lock = &bufmap->readdir_index_lock; |
| 505 | slargs.slot_wq = &readdir_waitq; |
| 506 | put_back_slot(&slargs, buffer_index); |
| 507 | pvfs2_bufmap_unref(bufmap); |
| 508 | } |
| 509 | |
| 510 | /* |
| 511 | * pvfs_bufmap_copy_iovec_from_user() |
| 512 | * |
| 513 | * copies data from several user space address's in an iovec |
| 514 | * to a mapped buffer |
| 515 | * |
| 516 | * Note that the mapped buffer is a series of pages and therefore |
| 517 | * the copies have to be split by PAGE_SIZE bytes at a time. |
| 518 | * Note that this routine checks that summation of iov_len |
| 519 | * across all the elements of iov is equal to size. |
| 520 | * |
| 521 | * returns 0 on success, -errno on failure |
| 522 | */ |
| 523 | int pvfs_bufmap_copy_iovec_from_user(struct pvfs2_bufmap *bufmap, |
| 524 | int buffer_index, |
| 525 | const struct iovec *iov, |
| 526 | unsigned long nr_segs, |
| 527 | size_t size) |
| 528 | { |
| 529 | size_t ret = 0; |
| 530 | size_t amt_copied = 0; |
| 531 | size_t cur_copy_size = 0; |
| 532 | unsigned int to_page_offset = 0; |
| 533 | unsigned int to_page_index = 0; |
| 534 | void *to_kaddr = NULL; |
| 535 | void __user *from_addr = NULL; |
| 536 | struct iovec *copied_iovec = NULL; |
| 537 | struct pvfs_bufmap_desc *to; |
| 538 | unsigned int seg; |
| 539 | char *tmp_printer = NULL; |
| 540 | int tmp_int = 0; |
| 541 | |
| 542 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 543 | "pvfs_bufmap_copy_iovec_from_user: index %d, " |
| 544 | "size %zd\n", |
| 545 | buffer_index, |
| 546 | size); |
| 547 | |
| 548 | to = &bufmap->desc_array[buffer_index]; |
| 549 | |
| 550 | /* |
| 551 | * copy the passed in iovec so that we can change some of its fields |
| 552 | */ |
| 553 | copied_iovec = kmalloc_array(nr_segs, |
| 554 | sizeof(*copied_iovec), |
| 555 | PVFS2_BUFMAP_GFP_FLAGS); |
| 556 | if (copied_iovec == NULL) |
| 557 | return -ENOMEM; |
| 558 | |
| 559 | memcpy(copied_iovec, iov, nr_segs * sizeof(*copied_iovec)); |
| 560 | /* |
| 561 | * Go through each segment in the iovec and make sure that |
| 562 | * the summation of iov_len matches the given size. |
| 563 | */ |
| 564 | for (seg = 0, amt_copied = 0; seg < nr_segs; seg++) |
| 565 | amt_copied += copied_iovec[seg].iov_len; |
| 566 | if (amt_copied != size) { |
| 567 | gossip_err( |
| 568 | "pvfs2_bufmap_copy_iovec_from_user: computed total (" |
| 569 | "%zd) is not equal to (%zd)\n", |
| 570 | amt_copied, |
| 571 | size); |
| 572 | kfree(copied_iovec); |
| 573 | return -EINVAL; |
| 574 | } |
| 575 | |
| 576 | to_page_index = 0; |
| 577 | to_page_offset = 0; |
| 578 | amt_copied = 0; |
| 579 | seg = 0; |
| 580 | /* |
| 581 | * Go through each segment in the iovec and copy its |
| 582 | * buffer into the mapped buffer one page at a time though |
| 583 | */ |
| 584 | while (amt_copied < size) { |
| 585 | struct iovec *iv = &copied_iovec[seg]; |
| 586 | int inc_to_page_index; |
| 587 | |
| 588 | if (iv->iov_len < (PAGE_SIZE - to_page_offset)) { |
| 589 | cur_copy_size = |
| 590 | PVFS_util_min(iv->iov_len, size - amt_copied); |
| 591 | seg++; |
| 592 | from_addr = iv->iov_base; |
| 593 | inc_to_page_index = 0; |
| 594 | } else if (iv->iov_len == (PAGE_SIZE - to_page_offset)) { |
| 595 | cur_copy_size = |
| 596 | PVFS_util_min(iv->iov_len, size - amt_copied); |
| 597 | seg++; |
| 598 | from_addr = iv->iov_base; |
| 599 | inc_to_page_index = 1; |
| 600 | } else { |
| 601 | cur_copy_size = |
| 602 | PVFS_util_min(PAGE_SIZE - to_page_offset, |
| 603 | size - amt_copied); |
| 604 | from_addr = iv->iov_base; |
| 605 | iv->iov_base += cur_copy_size; |
| 606 | iv->iov_len -= cur_copy_size; |
| 607 | inc_to_page_index = 1; |
| 608 | } |
| 609 | to_kaddr = pvfs2_kmap(to->page_array[to_page_index]); |
| 610 | ret = |
| 611 | copy_from_user(to_kaddr + to_page_offset, |
| 612 | from_addr, |
| 613 | cur_copy_size); |
| 614 | if (!PageReserved(to->page_array[to_page_index])) |
| 615 | SetPageDirty(to->page_array[to_page_index]); |
| 616 | |
| 617 | if (!tmp_printer) { |
| 618 | tmp_printer = (char *)(to_kaddr + to_page_offset); |
| 619 | tmp_int += tmp_printer[0]; |
| 620 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 621 | "First character (integer value) in pvfs_bufmap_copy_from_user: %d\n", |
| 622 | tmp_int); |
| 623 | } |
| 624 | |
| 625 | pvfs2_kunmap(to->page_array[to_page_index]); |
| 626 | if (ret) { |
| 627 | gossip_err("Failed to copy data from user space\n"); |
| 628 | kfree(copied_iovec); |
| 629 | return -EFAULT; |
| 630 | } |
| 631 | |
| 632 | amt_copied += cur_copy_size; |
| 633 | if (inc_to_page_index) { |
| 634 | to_page_offset = 0; |
| 635 | to_page_index++; |
| 636 | } else { |
| 637 | to_page_offset += cur_copy_size; |
| 638 | } |
| 639 | } |
| 640 | kfree(copied_iovec); |
| 641 | return 0; |
| 642 | } |
| 643 | |
| 644 | /* |
| 645 | * pvfs_bufmap_copy_iovec_from_kernel() |
| 646 | * |
| 647 | * copies data from several kernel space address's in an iovec |
| 648 | * to a mapped buffer |
| 649 | * |
| 650 | * Note that the mapped buffer is a series of pages and therefore |
| 651 | * the copies have to be split by PAGE_SIZE bytes at a time. |
| 652 | * Note that this routine checks that summation of iov_len |
| 653 | * across all the elements of iov is equal to size. |
| 654 | * |
| 655 | * returns 0 on success, -errno on failure |
| 656 | */ |
| 657 | int pvfs_bufmap_copy_iovec_from_kernel(struct pvfs2_bufmap *bufmap, |
| 658 | int buffer_index, const struct iovec *iov, |
| 659 | unsigned long nr_segs, size_t size) |
| 660 | { |
| 661 | size_t amt_copied = 0; |
| 662 | size_t cur_copy_size = 0; |
| 663 | int to_page_index = 0; |
| 664 | void *to_kaddr = NULL; |
| 665 | void *from_kaddr = NULL; |
Mike Marshall | 84d0215 | 2015-07-28 13:27:51 -0400 | [diff] [blame] | 666 | struct kvec *iv = NULL; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 667 | struct iovec *copied_iovec = NULL; |
| 668 | struct pvfs_bufmap_desc *to; |
| 669 | unsigned int seg; |
| 670 | unsigned to_page_offset = 0; |
| 671 | |
| 672 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 673 | "pvfs_bufmap_copy_iovec_from_kernel: index %d, " |
| 674 | "size %zd\n", |
| 675 | buffer_index, |
| 676 | size); |
| 677 | |
| 678 | to = &bufmap->desc_array[buffer_index]; |
| 679 | /* |
| 680 | * copy the passed in iovec so that we can change some of its fields |
| 681 | */ |
| 682 | copied_iovec = kmalloc_array(nr_segs, |
| 683 | sizeof(*copied_iovec), |
| 684 | PVFS2_BUFMAP_GFP_FLAGS); |
| 685 | if (copied_iovec == NULL) |
| 686 | return -ENOMEM; |
| 687 | |
| 688 | memcpy(copied_iovec, iov, nr_segs * sizeof(*copied_iovec)); |
| 689 | /* |
| 690 | * Go through each segment in the iovec and make sure that |
| 691 | * the summation of iov_len matches the given size. |
| 692 | */ |
| 693 | for (seg = 0, amt_copied = 0; seg < nr_segs; seg++) |
| 694 | amt_copied += copied_iovec[seg].iov_len; |
| 695 | if (amt_copied != size) { |
| 696 | gossip_err("pvfs2_bufmap_copy_iovec_from_kernel: computed total(%zd) is not equal to (%zd)\n", |
| 697 | amt_copied, |
| 698 | size); |
| 699 | kfree(copied_iovec); |
| 700 | return -EINVAL; |
| 701 | } |
| 702 | |
| 703 | to_page_index = 0; |
| 704 | amt_copied = 0; |
| 705 | seg = 0; |
| 706 | to_page_offset = 0; |
| 707 | /* |
| 708 | * Go through each segment in the iovec and copy its |
| 709 | * buffer into the mapped buffer one page at a time though |
| 710 | */ |
| 711 | while (amt_copied < size) { |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 712 | int inc_to_page_index; |
| 713 | |
Mike Marshall | 84d0215 | 2015-07-28 13:27:51 -0400 | [diff] [blame] | 714 | iv = (struct kvec *) &copied_iovec[seg]; |
| 715 | |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 716 | if (iv->iov_len < (PAGE_SIZE - to_page_offset)) { |
| 717 | cur_copy_size = |
| 718 | PVFS_util_min(iv->iov_len, size - amt_copied); |
| 719 | seg++; |
| 720 | from_kaddr = iv->iov_base; |
| 721 | inc_to_page_index = 0; |
| 722 | } else if (iv->iov_len == (PAGE_SIZE - to_page_offset)) { |
| 723 | cur_copy_size = |
| 724 | PVFS_util_min(iv->iov_len, size - amt_copied); |
| 725 | seg++; |
| 726 | from_kaddr = iv->iov_base; |
| 727 | inc_to_page_index = 1; |
| 728 | } else { |
| 729 | cur_copy_size = |
| 730 | PVFS_util_min(PAGE_SIZE - to_page_offset, |
| 731 | size - amt_copied); |
| 732 | from_kaddr = iv->iov_base; |
| 733 | iv->iov_base += cur_copy_size; |
| 734 | iv->iov_len -= cur_copy_size; |
| 735 | inc_to_page_index = 1; |
| 736 | } |
| 737 | to_kaddr = pvfs2_kmap(to->page_array[to_page_index]); |
| 738 | memcpy(to_kaddr + to_page_offset, from_kaddr, cur_copy_size); |
| 739 | if (!PageReserved(to->page_array[to_page_index])) |
| 740 | SetPageDirty(to->page_array[to_page_index]); |
| 741 | pvfs2_kunmap(to->page_array[to_page_index]); |
| 742 | amt_copied += cur_copy_size; |
| 743 | if (inc_to_page_index) { |
| 744 | to_page_offset = 0; |
| 745 | to_page_index++; |
| 746 | } else { |
| 747 | to_page_offset += cur_copy_size; |
| 748 | } |
| 749 | } |
| 750 | kfree(copied_iovec); |
| 751 | return 0; |
| 752 | } |
| 753 | |
| 754 | /* |
| 755 | * pvfs_bufmap_copy_to_user_iovec() |
| 756 | * |
| 757 | * copies data to several user space address's in an iovec |
| 758 | * from a mapped buffer |
| 759 | * |
| 760 | * returns 0 on success, -errno on failure |
| 761 | */ |
| 762 | int pvfs_bufmap_copy_to_user_iovec(struct pvfs2_bufmap *bufmap, |
| 763 | int buffer_index, const struct iovec *iov, |
| 764 | unsigned long nr_segs, size_t size) |
| 765 | { |
| 766 | size_t ret = 0; |
| 767 | size_t amt_copied = 0; |
| 768 | size_t cur_copy_size = 0; |
| 769 | int from_page_index = 0; |
| 770 | void *from_kaddr = NULL; |
| 771 | void __user *to_addr = NULL; |
| 772 | struct iovec *copied_iovec = NULL; |
| 773 | struct pvfs_bufmap_desc *from; |
| 774 | unsigned int seg; |
| 775 | unsigned from_page_offset = 0; |
| 776 | char *tmp_printer = NULL; |
| 777 | int tmp_int = 0; |
| 778 | |
| 779 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 780 | "pvfs_bufmap_copy_to_user_iovec: index %d, size %zd\n", |
| 781 | buffer_index, |
| 782 | size); |
| 783 | |
| 784 | from = &bufmap->desc_array[buffer_index]; |
| 785 | /* |
| 786 | * copy the passed in iovec so that we can change some of its fields |
| 787 | */ |
| 788 | copied_iovec = kmalloc_array(nr_segs, |
| 789 | sizeof(*copied_iovec), |
| 790 | PVFS2_BUFMAP_GFP_FLAGS); |
| 791 | if (copied_iovec == NULL) |
| 792 | return -ENOMEM; |
| 793 | |
| 794 | memcpy(copied_iovec, iov, nr_segs * sizeof(*copied_iovec)); |
| 795 | /* |
| 796 | * Go through each segment in the iovec and make sure that |
| 797 | * the summation of iov_len is greater than the given size. |
| 798 | */ |
| 799 | for (seg = 0, amt_copied = 0; seg < nr_segs; seg++) |
| 800 | amt_copied += copied_iovec[seg].iov_len; |
| 801 | if (amt_copied < size) { |
| 802 | gossip_err("pvfs2_bufmap_copy_to_user_iovec: computed total (%zd) is less than (%zd)\n", |
| 803 | amt_copied, |
| 804 | size); |
| 805 | kfree(copied_iovec); |
| 806 | return -EINVAL; |
| 807 | } |
| 808 | |
| 809 | from_page_index = 0; |
| 810 | amt_copied = 0; |
| 811 | seg = 0; |
| 812 | from_page_offset = 0; |
| 813 | /* |
| 814 | * Go through each segment in the iovec and copy from the mapper buffer, |
| 815 | * but make sure that we do so one page at a time. |
| 816 | */ |
| 817 | while (amt_copied < size) { |
| 818 | struct iovec *iv = &copied_iovec[seg]; |
| 819 | int inc_from_page_index; |
| 820 | |
| 821 | if (iv->iov_len < (PAGE_SIZE - from_page_offset)) { |
| 822 | cur_copy_size = |
| 823 | PVFS_util_min(iv->iov_len, size - amt_copied); |
| 824 | seg++; |
| 825 | to_addr = iv->iov_base; |
| 826 | inc_from_page_index = 0; |
| 827 | } else if (iv->iov_len == (PAGE_SIZE - from_page_offset)) { |
| 828 | cur_copy_size = |
| 829 | PVFS_util_min(iv->iov_len, size - amt_copied); |
| 830 | seg++; |
| 831 | to_addr = iv->iov_base; |
| 832 | inc_from_page_index = 1; |
| 833 | } else { |
| 834 | cur_copy_size = |
| 835 | PVFS_util_min(PAGE_SIZE - from_page_offset, |
| 836 | size - amt_copied); |
| 837 | to_addr = iv->iov_base; |
| 838 | iv->iov_base += cur_copy_size; |
| 839 | iv->iov_len -= cur_copy_size; |
| 840 | inc_from_page_index = 1; |
| 841 | } |
| 842 | from_kaddr = pvfs2_kmap(from->page_array[from_page_index]); |
| 843 | if (!tmp_printer) { |
| 844 | tmp_printer = (char *)(from_kaddr + from_page_offset); |
| 845 | tmp_int += tmp_printer[0]; |
| 846 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 847 | "First character (integer value) in pvfs_bufmap_copy_to_user_iovec: %d\n", |
| 848 | tmp_int); |
| 849 | } |
| 850 | ret = |
| 851 | copy_to_user(to_addr, |
| 852 | from_kaddr + from_page_offset, |
| 853 | cur_copy_size); |
| 854 | pvfs2_kunmap(from->page_array[from_page_index]); |
| 855 | if (ret) { |
| 856 | gossip_err("Failed to copy data to user space\n"); |
| 857 | kfree(copied_iovec); |
| 858 | return -EFAULT; |
| 859 | } |
| 860 | |
| 861 | amt_copied += cur_copy_size; |
| 862 | if (inc_from_page_index) { |
| 863 | from_page_offset = 0; |
| 864 | from_page_index++; |
| 865 | } else { |
| 866 | from_page_offset += cur_copy_size; |
| 867 | } |
| 868 | } |
| 869 | kfree(copied_iovec); |
| 870 | return 0; |
| 871 | } |
| 872 | |
| 873 | /* |
| 874 | * pvfs_bufmap_copy_to_kernel_iovec() |
| 875 | * |
| 876 | * copies data to several kernel space address's in an iovec |
| 877 | * from a mapped buffer |
| 878 | * |
| 879 | * returns 0 on success, -errno on failure |
| 880 | */ |
| 881 | int pvfs_bufmap_copy_to_kernel_iovec(struct pvfs2_bufmap *bufmap, |
| 882 | int buffer_index, const struct iovec *iov, |
| 883 | unsigned long nr_segs, size_t size) |
| 884 | { |
| 885 | size_t amt_copied = 0; |
| 886 | size_t cur_copy_size = 0; |
| 887 | int from_page_index = 0; |
| 888 | void *from_kaddr = NULL; |
| 889 | void *to_kaddr = NULL; |
Mike Marshall | 84d0215 | 2015-07-28 13:27:51 -0400 | [diff] [blame] | 890 | struct kvec *iv; |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 891 | struct iovec *copied_iovec = NULL; |
| 892 | struct pvfs_bufmap_desc *from; |
| 893 | unsigned int seg; |
| 894 | unsigned int from_page_offset = 0; |
| 895 | |
| 896 | gossip_debug(GOSSIP_BUFMAP_DEBUG, |
| 897 | "pvfs_bufmap_copy_to_kernel_iovec: index %d, size %zd\n", |
| 898 | buffer_index, |
| 899 | size); |
| 900 | |
| 901 | from = &bufmap->desc_array[buffer_index]; |
| 902 | /* |
| 903 | * copy the passed in iovec so that we can change some of its fields |
| 904 | */ |
| 905 | copied_iovec = kmalloc_array(nr_segs, |
| 906 | sizeof(*copied_iovec), |
| 907 | PVFS2_BUFMAP_GFP_FLAGS); |
| 908 | if (copied_iovec == NULL) |
| 909 | return -ENOMEM; |
| 910 | |
| 911 | memcpy(copied_iovec, iov, nr_segs * sizeof(*copied_iovec)); |
| 912 | /* |
| 913 | * Go through each segment in the iovec and make sure that |
| 914 | * the summation of iov_len is greater than the given size. |
| 915 | */ |
| 916 | for (seg = 0, amt_copied = 0; seg < nr_segs; seg++) |
| 917 | amt_copied += copied_iovec[seg].iov_len; |
| 918 | |
| 919 | if (amt_copied < size) { |
| 920 | gossip_err("pvfs2_bufmap_copy_to_kernel_iovec: computed total (%zd) is less than (%zd)\n", |
| 921 | amt_copied, |
| 922 | size); |
| 923 | kfree(copied_iovec); |
| 924 | return -EINVAL; |
| 925 | } |
| 926 | |
| 927 | from_page_index = 0; |
| 928 | amt_copied = 0; |
| 929 | seg = 0; |
| 930 | from_page_offset = 0; |
| 931 | /* |
| 932 | * Go through each segment in the iovec and copy from the mapper buffer, |
| 933 | * but make sure that we do so one page at a time. |
| 934 | */ |
| 935 | while (amt_copied < size) { |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 936 | int inc_from_page_index; |
| 937 | |
Mike Marshall | 84d0215 | 2015-07-28 13:27:51 -0400 | [diff] [blame] | 938 | iv = (struct kvec *) &copied_iovec[seg]; |
| 939 | |
Mike Marshall | 274dcf5 | 2015-07-17 10:38:13 -0400 | [diff] [blame] | 940 | if (iv->iov_len < (PAGE_SIZE - from_page_offset)) { |
| 941 | cur_copy_size = |
| 942 | PVFS_util_min(iv->iov_len, size - amt_copied); |
| 943 | seg++; |
| 944 | to_kaddr = iv->iov_base; |
| 945 | inc_from_page_index = 0; |
| 946 | } else if (iv->iov_len == (PAGE_SIZE - from_page_offset)) { |
| 947 | cur_copy_size = |
| 948 | PVFS_util_min(iv->iov_len, size - amt_copied); |
| 949 | seg++; |
| 950 | to_kaddr = iv->iov_base; |
| 951 | inc_from_page_index = 1; |
| 952 | } else { |
| 953 | cur_copy_size = |
| 954 | PVFS_util_min(PAGE_SIZE - from_page_offset, |
| 955 | size - amt_copied); |
| 956 | to_kaddr = iv->iov_base; |
| 957 | iv->iov_base += cur_copy_size; |
| 958 | iv->iov_len -= cur_copy_size; |
| 959 | inc_from_page_index = 1; |
| 960 | } |
| 961 | from_kaddr = pvfs2_kmap(from->page_array[from_page_index]); |
| 962 | memcpy(to_kaddr, from_kaddr + from_page_offset, cur_copy_size); |
| 963 | pvfs2_kunmap(from->page_array[from_page_index]); |
| 964 | amt_copied += cur_copy_size; |
| 965 | if (inc_from_page_index) { |
| 966 | from_page_offset = 0; |
| 967 | from_page_index++; |
| 968 | } else { |
| 969 | from_page_offset += cur_copy_size; |
| 970 | } |
| 971 | } |
| 972 | kfree(copied_iovec); |
| 973 | return 0; |
| 974 | } |