blob: 980ef13b4cf68194e3d9bb8a40134eb8796250ce [file] [log] [blame]
Sudeep Duttba612aa2015-09-29 18:15:29 -07001/*
2 * Intel MIC Platform Software Stack (MPSS)
3 *
4 * Copyright(c) 2015 Intel Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License, version 2, as
8 * published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License for more details.
14 *
15 * Intel SCIF driver.
16 *
17 */
18#include <linux/dma_remapping.h>
19#include <linux/pagemap.h>
20#include "scif_main.h"
21#include "scif_map.h"
22
23/* Used to skip ulimit checks for registrations with SCIF_MAP_KERNEL flag */
24#define SCIF_MAP_ULIMIT 0x40
25
26bool scif_ulimit_check = 1;
27
28/**
29 * scif_rma_ep_init:
30 * @ep: end point
31 *
32 * Initialize RMA per EP data structures.
33 */
34void scif_rma_ep_init(struct scif_endpt *ep)
35{
36 struct scif_endpt_rma_info *rma = &ep->rma_info;
37
38 mutex_init(&rma->rma_lock);
39 init_iova_domain(&rma->iovad, PAGE_SIZE, SCIF_IOVA_START_PFN,
40 SCIF_DMA_64BIT_PFN);
41 spin_lock_init(&rma->tc_lock);
42 mutex_init(&rma->mmn_lock);
43 INIT_LIST_HEAD(&rma->reg_list);
44 INIT_LIST_HEAD(&rma->remote_reg_list);
45 atomic_set(&rma->tw_refcount, 0);
46 atomic_set(&rma->tcw_refcount, 0);
47 atomic_set(&rma->tcw_total_pages, 0);
48 atomic_set(&rma->fence_refcount, 0);
49
50 rma->async_list_del = 0;
51 rma->dma_chan = NULL;
52 INIT_LIST_HEAD(&rma->mmn_list);
53 INIT_LIST_HEAD(&rma->vma_list);
54 init_waitqueue_head(&rma->markwq);
55}
56
57/**
58 * scif_rma_ep_can_uninit:
59 * @ep: end point
60 *
61 * Returns 1 if an endpoint can be uninitialized and 0 otherwise.
62 */
63int scif_rma_ep_can_uninit(struct scif_endpt *ep)
64{
65 int ret = 0;
66
67 mutex_lock(&ep->rma_info.rma_lock);
68 /* Destroy RMA Info only if both lists are empty */
69 if (list_empty(&ep->rma_info.reg_list) &&
70 list_empty(&ep->rma_info.remote_reg_list) &&
71 list_empty(&ep->rma_info.mmn_list) &&
72 !atomic_read(&ep->rma_info.tw_refcount) &&
73 !atomic_read(&ep->rma_info.tcw_refcount) &&
74 !atomic_read(&ep->rma_info.fence_refcount))
75 ret = 1;
76 mutex_unlock(&ep->rma_info.rma_lock);
77 return ret;
78}
79
80/**
81 * scif_create_pinned_pages:
82 * @nr_pages: number of pages in window
83 * @prot: read/write protection
84 *
85 * Allocate and prepare a set of pinned pages.
86 */
87static struct scif_pinned_pages *
88scif_create_pinned_pages(int nr_pages, int prot)
89{
90 struct scif_pinned_pages *pin;
91
92 might_sleep();
93 pin = scif_zalloc(sizeof(*pin));
94 if (!pin)
95 goto error;
96
97 pin->pages = scif_zalloc(nr_pages * sizeof(*pin->pages));
98 if (!pin->pages)
99 goto error_free_pinned_pages;
100
101 pin->prot = prot;
102 pin->magic = SCIFEP_MAGIC;
103 return pin;
104
105error_free_pinned_pages:
106 scif_free(pin, sizeof(*pin));
107error:
108 return NULL;
109}
110
111/**
112 * scif_destroy_pinned_pages:
113 * @pin: A set of pinned pages.
114 *
115 * Deallocate resources for pinned pages.
116 */
117static int scif_destroy_pinned_pages(struct scif_pinned_pages *pin)
118{
119 int j;
120 int writeable = pin->prot & SCIF_PROT_WRITE;
121 int kernel = SCIF_MAP_KERNEL & pin->map_flags;
122
123 for (j = 0; j < pin->nr_pages; j++) {
124 if (pin->pages[j] && !kernel) {
125 if (writeable)
126 SetPageDirty(pin->pages[j]);
127 put_page(pin->pages[j]);
128 }
129 }
130
131 scif_free(pin->pages,
132 pin->nr_pages * sizeof(*pin->pages));
133 scif_free(pin, sizeof(*pin));
134 return 0;
135}
136
137/*
138 * scif_create_window:
139 * @ep: end point
140 * @nr_pages: number of pages
141 * @offset: registration offset
142 * @temp: true if a temporary window is being created
143 *
144 * Allocate and prepare a self registration window.
145 */
146struct scif_window *scif_create_window(struct scif_endpt *ep, int nr_pages,
147 s64 offset, bool temp)
148{
149 struct scif_window *window;
150
151 might_sleep();
152 window = scif_zalloc(sizeof(*window));
153 if (!window)
154 goto error;
155
156 window->dma_addr = scif_zalloc(nr_pages * sizeof(*window->dma_addr));
157 if (!window->dma_addr)
158 goto error_free_window;
159
160 window->num_pages = scif_zalloc(nr_pages * sizeof(*window->num_pages));
161 if (!window->num_pages)
162 goto error_free_window;
163
164 window->offset = offset;
165 window->ep = (u64)ep;
166 window->magic = SCIFEP_MAGIC;
167 window->reg_state = OP_IDLE;
168 init_waitqueue_head(&window->regwq);
169 window->unreg_state = OP_IDLE;
170 init_waitqueue_head(&window->unregwq);
171 INIT_LIST_HEAD(&window->list);
172 window->type = SCIF_WINDOW_SELF;
173 window->temp = temp;
174 return window;
175
176error_free_window:
177 scif_free(window->dma_addr,
178 nr_pages * sizeof(*window->dma_addr));
179 scif_free(window, sizeof(*window));
180error:
181 return NULL;
182}
183
184/**
185 * scif_destroy_incomplete_window:
186 * @ep: end point
187 * @window: registration window
188 *
189 * Deallocate resources for self window.
190 */
191static void scif_destroy_incomplete_window(struct scif_endpt *ep,
192 struct scif_window *window)
193{
194 int err;
195 int nr_pages = window->nr_pages;
196 struct scif_allocmsg *alloc = &window->alloc_handle;
197 struct scifmsg msg;
198
199retry:
200 /* Wait for a SCIF_ALLOC_GNT/REJ message */
201 err = wait_event_timeout(alloc->allocwq,
202 alloc->state != OP_IN_PROGRESS,
203 SCIF_NODE_ALIVE_TIMEOUT);
204 if (!err && scifdev_alive(ep))
205 goto retry;
206
207 mutex_lock(&ep->rma_info.rma_lock);
208 if (alloc->state == OP_COMPLETED) {
209 msg.uop = SCIF_FREE_VIRT;
210 msg.src = ep->port;
211 msg.payload[0] = ep->remote_ep;
212 msg.payload[1] = window->alloc_handle.vaddr;
213 msg.payload[2] = (u64)window;
214 msg.payload[3] = SCIF_REGISTER;
215 _scif_nodeqp_send(ep->remote_dev, &msg);
216 }
217 mutex_unlock(&ep->rma_info.rma_lock);
218
219 scif_free_window_offset(ep, window, window->offset);
220 scif_free(window->dma_addr, nr_pages * sizeof(*window->dma_addr));
221 scif_free(window->num_pages, nr_pages * sizeof(*window->num_pages));
222 scif_free(window, sizeof(*window));
223}
224
225/**
226 * scif_unmap_window:
227 * @remote_dev: SCIF remote device
228 * @window: registration window
229 *
230 * Delete any DMA mappings created for a registered self window
231 */
232void scif_unmap_window(struct scif_dev *remote_dev, struct scif_window *window)
233{
234 int j;
235
236 if (scif_is_iommu_enabled() && !scifdev_self(remote_dev)) {
237 if (window->st) {
238 dma_unmap_sg(&remote_dev->sdev->dev,
239 window->st->sgl, window->st->nents,
240 DMA_BIDIRECTIONAL);
241 sg_free_table(window->st);
242 kfree(window->st);
243 window->st = NULL;
244 }
245 } else {
246 for (j = 0; j < window->nr_contig_chunks; j++) {
247 if (window->dma_addr[j]) {
248 scif_unmap_single(window->dma_addr[j],
249 remote_dev,
250 window->num_pages[j] <<
251 PAGE_SHIFT);
252 window->dma_addr[j] = 0x0;
253 }
254 }
255 }
256}
257
258static inline struct mm_struct *__scif_acquire_mm(void)
259{
260 if (scif_ulimit_check)
261 return get_task_mm(current);
262 return NULL;
263}
264
265static inline void __scif_release_mm(struct mm_struct *mm)
266{
267 if (mm)
268 mmput(mm);
269}
270
271static inline int
272__scif_dec_pinned_vm_lock(struct mm_struct *mm,
273 int nr_pages, bool try_lock)
274{
275 if (!mm || !nr_pages || !scif_ulimit_check)
276 return 0;
277 if (try_lock) {
278 if (!down_write_trylock(&mm->mmap_sem)) {
279 dev_err(scif_info.mdev.this_device,
280 "%s %d err\n", __func__, __LINE__);
281 return -1;
282 }
283 } else {
284 down_write(&mm->mmap_sem);
285 }
286 mm->pinned_vm -= nr_pages;
287 up_write(&mm->mmap_sem);
288 return 0;
289}
290
291static inline int __scif_check_inc_pinned_vm(struct mm_struct *mm,
292 int nr_pages)
293{
294 unsigned long locked, lock_limit;
295
296 if (!mm || !nr_pages || !scif_ulimit_check)
297 return 0;
298
299 locked = nr_pages;
300 locked += mm->pinned_vm;
301 lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
302 if ((locked > lock_limit) && !capable(CAP_IPC_LOCK)) {
303 dev_err(scif_info.mdev.this_device,
304 "locked(%lu) > lock_limit(%lu)\n",
305 locked, lock_limit);
306 return -ENOMEM;
307 }
308 mm->pinned_vm = locked;
309 return 0;
310}
311
312/**
313 * scif_destroy_window:
314 * @ep: end point
315 * @window: registration window
316 *
317 * Deallocate resources for self window.
318 */
319int scif_destroy_window(struct scif_endpt *ep, struct scif_window *window)
320{
321 int j;
322 struct scif_pinned_pages *pinned_pages = window->pinned_pages;
323 int nr_pages = window->nr_pages;
324
325 might_sleep();
326 if (!window->temp && window->mm) {
327 __scif_dec_pinned_vm_lock(window->mm, window->nr_pages, 0);
328 __scif_release_mm(window->mm);
329 window->mm = NULL;
330 }
331
332 scif_free_window_offset(ep, window, window->offset);
333 scif_unmap_window(ep->remote_dev, window);
334 /*
335 * Decrement references for this set of pinned pages from
336 * this window.
337 */
338 j = atomic_sub_return(1, &pinned_pages->ref_count);
339 if (j < 0)
340 dev_err(scif_info.mdev.this_device,
341 "%s %d incorrect ref count %d\n",
342 __func__, __LINE__, j);
343 /*
344 * If the ref count for pinned_pages is zero then someone
345 * has already called scif_unpin_pages() for it and we should
346 * destroy the page cache.
347 */
348 if (!j)
349 scif_destroy_pinned_pages(window->pinned_pages);
350 scif_free(window->dma_addr, nr_pages * sizeof(*window->dma_addr));
351 scif_free(window->num_pages, nr_pages * sizeof(*window->num_pages));
352 window->magic = 0;
353 scif_free(window, sizeof(*window));
354 return 0;
355}
356
357/**
358 * scif_create_remote_lookup:
359 * @remote_dev: SCIF remote device
360 * @window: remote window
361 *
362 * Allocate and prepare lookup entries for the remote
363 * end to copy over the physical addresses.
364 * Returns 0 on success and appropriate errno on failure.
365 */
366static int scif_create_remote_lookup(struct scif_dev *remote_dev,
367 struct scif_window *window)
368{
369 int i, j, err = 0;
370 int nr_pages = window->nr_pages;
371 bool vmalloc_dma_phys, vmalloc_num_pages;
372
373 might_sleep();
374 /* Map window */
375 err = scif_map_single(&window->mapped_offset,
376 window, remote_dev, sizeof(*window));
377 if (err)
378 goto error_window;
379
380 /* Compute the number of lookup entries. 21 == 2MB Shift */
381 window->nr_lookup = ALIGN(nr_pages * PAGE_SIZE,
382 ((2) * 1024 * 1024)) >> 21;
383
384 window->dma_addr_lookup.lookup =
385 scif_alloc_coherent(&window->dma_addr_lookup.offset,
386 remote_dev, window->nr_lookup *
387 sizeof(*window->dma_addr_lookup.lookup),
388 GFP_KERNEL | __GFP_ZERO);
389 if (!window->dma_addr_lookup.lookup)
390 goto error_window;
391
392 window->num_pages_lookup.lookup =
393 scif_alloc_coherent(&window->num_pages_lookup.offset,
394 remote_dev, window->nr_lookup *
395 sizeof(*window->num_pages_lookup.lookup),
396 GFP_KERNEL | __GFP_ZERO);
397 if (!window->num_pages_lookup.lookup)
398 goto error_window;
399
400 vmalloc_dma_phys = is_vmalloc_addr(&window->dma_addr[0]);
401 vmalloc_num_pages = is_vmalloc_addr(&window->num_pages[0]);
402
403 /* Now map each of the pages containing physical addresses */
404 for (i = 0, j = 0; i < nr_pages; i += SCIF_NR_ADDR_IN_PAGE, j++) {
405 err = scif_map_page(&window->dma_addr_lookup.lookup[j],
406 vmalloc_dma_phys ?
407 vmalloc_to_page(&window->dma_addr[i]) :
408 virt_to_page(&window->dma_addr[i]),
409 remote_dev);
410 if (err)
411 goto error_window;
412 err = scif_map_page(&window->num_pages_lookup.lookup[j],
413 vmalloc_dma_phys ?
414 vmalloc_to_page(&window->num_pages[i]) :
415 virt_to_page(&window->num_pages[i]),
416 remote_dev);
417 if (err)
418 goto error_window;
419 }
420 return 0;
421error_window:
422 return err;
423}
424
425/**
426 * scif_destroy_remote_lookup:
427 * @remote_dev: SCIF remote device
428 * @window: remote window
429 *
430 * Destroy lookup entries used for the remote
431 * end to copy over the physical addresses.
432 */
433static void scif_destroy_remote_lookup(struct scif_dev *remote_dev,
434 struct scif_window *window)
435{
436 int i, j;
437
438 if (window->nr_lookup) {
439 struct scif_rma_lookup *lup = &window->dma_addr_lookup;
440 struct scif_rma_lookup *npup = &window->num_pages_lookup;
441
442 for (i = 0, j = 0; i < window->nr_pages;
443 i += SCIF_NR_ADDR_IN_PAGE, j++) {
444 if (lup->lookup && lup->lookup[j])
445 scif_unmap_single(lup->lookup[j],
446 remote_dev,
447 PAGE_SIZE);
448 if (npup->lookup && npup->lookup[j])
449 scif_unmap_single(npup->lookup[j],
450 remote_dev,
451 PAGE_SIZE);
452 }
453 if (lup->lookup)
454 scif_free_coherent(lup->lookup, lup->offset,
455 remote_dev, window->nr_lookup *
456 sizeof(*lup->lookup));
457 if (npup->lookup)
458 scif_free_coherent(npup->lookup, npup->offset,
459 remote_dev, window->nr_lookup *
460 sizeof(*npup->lookup));
461 if (window->mapped_offset)
462 scif_unmap_single(window->mapped_offset,
463 remote_dev, sizeof(*window));
464 window->nr_lookup = 0;
465 }
466}
467
468/**
469 * scif_create_remote_window:
470 * @ep: end point
471 * @nr_pages: number of pages in window
472 *
473 * Allocate and prepare a remote registration window.
474 */
475static struct scif_window *
476scif_create_remote_window(struct scif_dev *scifdev, int nr_pages)
477{
478 struct scif_window *window;
479
480 might_sleep();
481 window = scif_zalloc(sizeof(*window));
482 if (!window)
483 goto error_ret;
484
485 window->magic = SCIFEP_MAGIC;
486 window->nr_pages = nr_pages;
487
488 window->dma_addr = scif_zalloc(nr_pages * sizeof(*window->dma_addr));
489 if (!window->dma_addr)
490 goto error_window;
491
492 window->num_pages = scif_zalloc(nr_pages *
493 sizeof(*window->num_pages));
494 if (!window->num_pages)
495 goto error_window;
496
497 if (scif_create_remote_lookup(scifdev, window))
498 goto error_window;
499
500 window->type = SCIF_WINDOW_PEER;
501 window->unreg_state = OP_IDLE;
502 INIT_LIST_HEAD(&window->list);
503 return window;
504error_window:
505 scif_destroy_remote_window(window);
506error_ret:
507 return NULL;
508}
509
510/**
511 * scif_destroy_remote_window:
512 * @ep: end point
513 * @window: remote registration window
514 *
515 * Deallocate resources for remote window.
516 */
517void
518scif_destroy_remote_window(struct scif_window *window)
519{
520 scif_free(window->dma_addr, window->nr_pages *
521 sizeof(*window->dma_addr));
522 scif_free(window->num_pages, window->nr_pages *
523 sizeof(*window->num_pages));
524 window->magic = 0;
525 scif_free(window, sizeof(*window));
526}
527
528/**
529 * scif_iommu_map: create DMA mappings if the IOMMU is enabled
530 * @remote_dev: SCIF remote device
531 * @window: remote registration window
532 *
533 * Map the physical pages using dma_map_sg(..) and then detect the number
534 * of contiguous DMA mappings allocated
535 */
536static int scif_iommu_map(struct scif_dev *remote_dev,
537 struct scif_window *window)
538{
539 struct scatterlist *sg;
540 int i, err;
541 scif_pinned_pages_t pin = window->pinned_pages;
542
543 window->st = kzalloc(sizeof(*window->st), GFP_KERNEL);
544 if (!window->st)
545 return -ENOMEM;
546
547 err = sg_alloc_table(window->st, window->nr_pages, GFP_KERNEL);
548 if (err)
549 return err;
550
551 for_each_sg(window->st->sgl, sg, window->st->nents, i)
552 sg_set_page(sg, pin->pages[i], PAGE_SIZE, 0x0);
553
554 err = dma_map_sg(&remote_dev->sdev->dev, window->st->sgl,
555 window->st->nents, DMA_BIDIRECTIONAL);
556 if (!err)
557 return -ENOMEM;
558 /* Detect contiguous ranges of DMA mappings */
559 sg = window->st->sgl;
560 for (i = 0; sg; i++) {
561 dma_addr_t last_da;
562
563 window->dma_addr[i] = sg_dma_address(sg);
564 window->num_pages[i] = sg_dma_len(sg) >> PAGE_SHIFT;
565 last_da = sg_dma_address(sg) + sg_dma_len(sg);
566 while ((sg = sg_next(sg)) && sg_dma_address(sg) == last_da) {
567 window->num_pages[i] +=
568 (sg_dma_len(sg) >> PAGE_SHIFT);
569 last_da = window->dma_addr[i] +
570 sg_dma_len(sg);
571 }
572 window->nr_contig_chunks++;
573 }
574 return 0;
575}
576
577/**
578 * scif_map_window:
579 * @remote_dev: SCIF remote device
580 * @window: self registration window
581 *
582 * Map pages of a window into the aperture/PCI.
583 * Also determine addresses required for DMA.
584 */
585int
586scif_map_window(struct scif_dev *remote_dev, struct scif_window *window)
587{
588 int i, j, k, err = 0, nr_contig_pages;
589 scif_pinned_pages_t pin;
590 phys_addr_t phys_prev, phys_curr;
591
592 might_sleep();
593
594 pin = window->pinned_pages;
595
596 if (intel_iommu_enabled && !scifdev_self(remote_dev))
597 return scif_iommu_map(remote_dev, window);
598
599 for (i = 0, j = 0; i < window->nr_pages; i += nr_contig_pages, j++) {
600 phys_prev = page_to_phys(pin->pages[i]);
601 nr_contig_pages = 1;
602
603 /* Detect physically contiguous chunks */
604 for (k = i + 1; k < window->nr_pages; k++) {
605 phys_curr = page_to_phys(pin->pages[k]);
606 if (phys_curr != (phys_prev + PAGE_SIZE))
607 break;
608 phys_prev = phys_curr;
609 nr_contig_pages++;
610 }
611 window->num_pages[j] = nr_contig_pages;
612 window->nr_contig_chunks++;
613 if (scif_is_mgmt_node()) {
614 /*
615 * Management node has to deal with SMPT on X100 and
616 * hence the DMA mapping is required
617 */
618 err = scif_map_single(&window->dma_addr[j],
619 phys_to_virt(page_to_phys(
620 pin->pages[i])),
621 remote_dev,
622 nr_contig_pages << PAGE_SHIFT);
623 if (err)
624 return err;
625 } else {
626 window->dma_addr[j] = page_to_phys(pin->pages[i]);
627 }
628 }
629 return err;
630}
631
632/**
633 * scif_send_scif_unregister:
634 * @ep: end point
635 * @window: self registration window
636 *
637 * Send a SCIF_UNREGISTER message.
638 */
639static int scif_send_scif_unregister(struct scif_endpt *ep,
640 struct scif_window *window)
641{
642 struct scifmsg msg;
643
644 msg.uop = SCIF_UNREGISTER;
645 msg.src = ep->port;
646 msg.payload[0] = window->alloc_handle.vaddr;
647 msg.payload[1] = (u64)window;
648 return scif_nodeqp_send(ep->remote_dev, &msg);
649}
650
651/**
652 * scif_unregister_window:
653 * @window: self registration window
654 *
655 * Send an unregistration request and wait for a response.
656 */
657int scif_unregister_window(struct scif_window *window)
658{
659 int err = 0;
660 struct scif_endpt *ep = (struct scif_endpt *)window->ep;
661 bool send_msg = false;
662
663 might_sleep();
664 switch (window->unreg_state) {
665 case OP_IDLE:
666 {
667 window->unreg_state = OP_IN_PROGRESS;
668 send_msg = true;
669 /* fall through */
670 }
671 case OP_IN_PROGRESS:
672 {
673 scif_get_window(window, 1);
674 mutex_unlock(&ep->rma_info.rma_lock);
675 if (send_msg) {
676 err = scif_send_scif_unregister(ep, window);
677 if (err) {
678 window->unreg_state = OP_COMPLETED;
679 goto done;
680 }
681 } else {
682 /* Return ENXIO since unregistration is in progress */
Dan Carpenterff652122015-10-13 15:52:06 +0300683 mutex_lock(&ep->rma_info.rma_lock);
Sudeep Duttba612aa2015-09-29 18:15:29 -0700684 return -ENXIO;
685 }
686retry:
687 /* Wait for a SCIF_UNREGISTER_(N)ACK message */
688 err = wait_event_timeout(window->unregwq,
689 window->unreg_state != OP_IN_PROGRESS,
690 SCIF_NODE_ALIVE_TIMEOUT);
691 if (!err && scifdev_alive(ep))
692 goto retry;
693 if (!err) {
694 err = -ENODEV;
695 window->unreg_state = OP_COMPLETED;
696 dev_err(scif_info.mdev.this_device,
697 "%s %d err %d\n", __func__, __LINE__, err);
698 }
699 if (err > 0)
700 err = 0;
701done:
702 mutex_lock(&ep->rma_info.rma_lock);
703 scif_put_window(window, 1);
704 break;
705 }
706 case OP_FAILED:
707 {
708 if (!scifdev_alive(ep)) {
709 err = -ENODEV;
710 window->unreg_state = OP_COMPLETED;
711 }
712 break;
713 }
714 case OP_COMPLETED:
715 break;
716 default:
717 err = -ENODEV;
718 }
719
720 if (window->unreg_state == OP_COMPLETED && window->ref_count)
721 scif_put_window(window, window->nr_pages);
722
723 if (!window->ref_count) {
724 atomic_inc(&ep->rma_info.tw_refcount);
725 list_del_init(&window->list);
726 scif_free_window_offset(ep, window, window->offset);
727 mutex_unlock(&ep->rma_info.rma_lock);
728 if ((!!(window->pinned_pages->map_flags & SCIF_MAP_KERNEL)) &&
729 scifdev_alive(ep)) {
730 scif_drain_dma_intr(ep->remote_dev->sdev,
731 ep->rma_info.dma_chan);
732 } else {
733 if (!__scif_dec_pinned_vm_lock(window->mm,
734 window->nr_pages, 1)) {
735 __scif_release_mm(window->mm);
736 window->mm = NULL;
737 }
738 }
739 scif_queue_for_cleanup(window, &scif_info.rma);
740 mutex_lock(&ep->rma_info.rma_lock);
741 }
742 return err;
743}
744
745/**
746 * scif_send_alloc_request:
747 * @ep: end point
748 * @window: self registration window
749 *
750 * Send a remote window allocation request
751 */
752static int scif_send_alloc_request(struct scif_endpt *ep,
753 struct scif_window *window)
754{
755 struct scifmsg msg;
756 struct scif_allocmsg *alloc = &window->alloc_handle;
757
758 /* Set up the Alloc Handle */
759 alloc->state = OP_IN_PROGRESS;
760 init_waitqueue_head(&alloc->allocwq);
761
762 /* Send out an allocation request */
763 msg.uop = SCIF_ALLOC_REQ;
764 msg.payload[1] = window->nr_pages;
765 msg.payload[2] = (u64)&window->alloc_handle;
766 return _scif_nodeqp_send(ep->remote_dev, &msg);
767}
768
769/**
770 * scif_prep_remote_window:
771 * @ep: end point
772 * @window: self registration window
773 *
774 * Send a remote window allocation request, wait for an allocation response,
775 * and prepares the remote window by copying over the page lists
776 */
777static int scif_prep_remote_window(struct scif_endpt *ep,
778 struct scif_window *window)
779{
780 struct scifmsg msg;
781 struct scif_window *remote_window;
782 struct scif_allocmsg *alloc = &window->alloc_handle;
783 dma_addr_t *dma_phys_lookup, *tmp, *num_pages_lookup, *tmp1;
784 int i = 0, j = 0;
785 int nr_contig_chunks, loop_nr_contig_chunks;
786 int remaining_nr_contig_chunks, nr_lookup;
787 int err, map_err;
788
789 map_err = scif_map_window(ep->remote_dev, window);
790 if (map_err)
791 dev_err(&ep->remote_dev->sdev->dev,
792 "%s %d map_err %d\n", __func__, __LINE__, map_err);
793 remaining_nr_contig_chunks = window->nr_contig_chunks;
794 nr_contig_chunks = window->nr_contig_chunks;
795retry:
796 /* Wait for a SCIF_ALLOC_GNT/REJ message */
797 err = wait_event_timeout(alloc->allocwq,
798 alloc->state != OP_IN_PROGRESS,
799 SCIF_NODE_ALIVE_TIMEOUT);
800 mutex_lock(&ep->rma_info.rma_lock);
801 /* Synchronize with the thread waking up allocwq */
802 mutex_unlock(&ep->rma_info.rma_lock);
803 if (!err && scifdev_alive(ep))
804 goto retry;
805
806 if (!err)
807 err = -ENODEV;
808
809 if (err > 0)
810 err = 0;
811 else
812 return err;
813
814 /* Bail out. The remote end rejected this request */
815 if (alloc->state == OP_FAILED)
816 return -ENOMEM;
817
818 if (map_err) {
819 dev_err(&ep->remote_dev->sdev->dev,
820 "%s %d err %d\n", __func__, __LINE__, map_err);
821 msg.uop = SCIF_FREE_VIRT;
822 msg.src = ep->port;
823 msg.payload[0] = ep->remote_ep;
824 msg.payload[1] = window->alloc_handle.vaddr;
825 msg.payload[2] = (u64)window;
826 msg.payload[3] = SCIF_REGISTER;
827 spin_lock(&ep->lock);
828 if (ep->state == SCIFEP_CONNECTED)
829 err = _scif_nodeqp_send(ep->remote_dev, &msg);
830 else
831 err = -ENOTCONN;
832 spin_unlock(&ep->lock);
833 return err;
834 }
835
836 remote_window = scif_ioremap(alloc->phys_addr, sizeof(*window),
837 ep->remote_dev);
838
839 /* Compute the number of lookup entries. 21 == 2MB Shift */
840 nr_lookup = ALIGN(nr_contig_chunks, SCIF_NR_ADDR_IN_PAGE)
841 >> ilog2(SCIF_NR_ADDR_IN_PAGE);
842
843 dma_phys_lookup =
844 scif_ioremap(remote_window->dma_addr_lookup.offset,
845 nr_lookup *
846 sizeof(*remote_window->dma_addr_lookup.lookup),
847 ep->remote_dev);
848 num_pages_lookup =
849 scif_ioremap(remote_window->num_pages_lookup.offset,
850 nr_lookup *
851 sizeof(*remote_window->num_pages_lookup.lookup),
852 ep->remote_dev);
853
854 while (remaining_nr_contig_chunks) {
855 loop_nr_contig_chunks = min_t(int, remaining_nr_contig_chunks,
856 (int)SCIF_NR_ADDR_IN_PAGE);
857 /* #1/2 - Copy physical addresses over to the remote side */
858
859 /* #2/2 - Copy DMA addresses (addresses that are fed into the
860 * DMA engine) We transfer bus addresses which are then
861 * converted into a MIC physical address on the remote
862 * side if it is a MIC, if the remote node is a mgmt node we
863 * transfer the MIC physical address
864 */
865 tmp = scif_ioremap(dma_phys_lookup[j],
866 loop_nr_contig_chunks *
867 sizeof(*window->dma_addr),
868 ep->remote_dev);
869 tmp1 = scif_ioremap(num_pages_lookup[j],
870 loop_nr_contig_chunks *
871 sizeof(*window->num_pages),
872 ep->remote_dev);
873 if (scif_is_mgmt_node()) {
874 memcpy_toio((void __force __iomem *)tmp,
875 &window->dma_addr[i], loop_nr_contig_chunks
876 * sizeof(*window->dma_addr));
877 memcpy_toio((void __force __iomem *)tmp1,
878 &window->num_pages[i], loop_nr_contig_chunks
879 * sizeof(*window->num_pages));
880 } else {
881 if (scifdev_is_p2p(ep->remote_dev)) {
882 /*
883 * add remote node's base address for this node
884 * to convert it into a MIC address
885 */
886 int m;
887 dma_addr_t dma_addr;
888
889 for (m = 0; m < loop_nr_contig_chunks; m++) {
890 dma_addr = window->dma_addr[i + m] +
891 ep->remote_dev->base_addr;
892 writeq(dma_addr,
893 (void __force __iomem *)&tmp[m]);
894 }
895 memcpy_toio((void __force __iomem *)tmp1,
896 &window->num_pages[i],
897 loop_nr_contig_chunks
898 * sizeof(*window->num_pages));
899 } else {
900 /* Mgmt node or loopback - transfer DMA
901 * addresses as is, this is the same as a
902 * MIC physical address (we use the dma_addr
903 * and not the phys_addr array since the
904 * phys_addr is only setup if there is a mmap()
905 * request from the mgmt node)
906 */
907 memcpy_toio((void __force __iomem *)tmp,
908 &window->dma_addr[i],
909 loop_nr_contig_chunks *
910 sizeof(*window->dma_addr));
911 memcpy_toio((void __force __iomem *)tmp1,
912 &window->num_pages[i],
913 loop_nr_contig_chunks *
914 sizeof(*window->num_pages));
915 }
916 }
917 remaining_nr_contig_chunks -= loop_nr_contig_chunks;
918 i += loop_nr_contig_chunks;
919 j++;
920 scif_iounmap(tmp, loop_nr_contig_chunks *
921 sizeof(*window->dma_addr), ep->remote_dev);
922 scif_iounmap(tmp1, loop_nr_contig_chunks *
923 sizeof(*window->num_pages), ep->remote_dev);
924 }
925
926 /* Prepare the remote window for the peer */
927 remote_window->peer_window = (u64)window;
928 remote_window->offset = window->offset;
929 remote_window->prot = window->prot;
930 remote_window->nr_contig_chunks = nr_contig_chunks;
931 remote_window->ep = ep->remote_ep;
932 scif_iounmap(num_pages_lookup,
933 nr_lookup *
934 sizeof(*remote_window->num_pages_lookup.lookup),
935 ep->remote_dev);
936 scif_iounmap(dma_phys_lookup,
937 nr_lookup *
938 sizeof(*remote_window->dma_addr_lookup.lookup),
939 ep->remote_dev);
940 scif_iounmap(remote_window, sizeof(*remote_window), ep->remote_dev);
941 window->peer_window = alloc->vaddr;
942 return err;
943}
944
945/**
946 * scif_send_scif_register:
947 * @ep: end point
948 * @window: self registration window
949 *
950 * Send a SCIF_REGISTER message if EP is connected and wait for a
951 * SCIF_REGISTER_(N)ACK message else send a SCIF_FREE_VIRT
952 * message so that the peer can free its remote window allocated earlier.
953 */
954static int scif_send_scif_register(struct scif_endpt *ep,
955 struct scif_window *window)
956{
957 int err = 0;
958 struct scifmsg msg;
959
960 msg.src = ep->port;
961 msg.payload[0] = ep->remote_ep;
962 msg.payload[1] = window->alloc_handle.vaddr;
963 msg.payload[2] = (u64)window;
964 spin_lock(&ep->lock);
965 if (ep->state == SCIFEP_CONNECTED) {
966 msg.uop = SCIF_REGISTER;
967 window->reg_state = OP_IN_PROGRESS;
968 err = _scif_nodeqp_send(ep->remote_dev, &msg);
969 spin_unlock(&ep->lock);
970 if (!err) {
971retry:
972 /* Wait for a SCIF_REGISTER_(N)ACK message */
973 err = wait_event_timeout(window->regwq,
974 window->reg_state !=
975 OP_IN_PROGRESS,
976 SCIF_NODE_ALIVE_TIMEOUT);
977 if (!err && scifdev_alive(ep))
978 goto retry;
979 err = !err ? -ENODEV : 0;
980 if (window->reg_state == OP_FAILED)
981 err = -ENOTCONN;
982 }
983 } else {
984 msg.uop = SCIF_FREE_VIRT;
985 msg.payload[3] = SCIF_REGISTER;
986 err = _scif_nodeqp_send(ep->remote_dev, &msg);
987 spin_unlock(&ep->lock);
988 if (!err)
989 err = -ENOTCONN;
990 }
991 return err;
992}
993
994/**
995 * scif_get_window_offset:
996 * @ep: end point descriptor
997 * @flags: flags
998 * @offset: offset hint
999 * @num_pages: number of pages
1000 * @out_offset: computed offset returned by reference.
1001 *
1002 * Compute/Claim a new offset for this EP.
1003 */
1004int scif_get_window_offset(struct scif_endpt *ep, int flags, s64 offset,
1005 int num_pages, s64 *out_offset)
1006{
1007 s64 page_index;
1008 struct iova *iova_ptr;
1009 int err = 0;
1010
1011 if (flags & SCIF_MAP_FIXED) {
1012 page_index = SCIF_IOVA_PFN(offset);
1013 iova_ptr = reserve_iova(&ep->rma_info.iovad, page_index,
1014 page_index + num_pages - 1);
1015 if (!iova_ptr)
1016 err = -EADDRINUSE;
1017 } else {
1018 iova_ptr = alloc_iova(&ep->rma_info.iovad, num_pages,
1019 SCIF_DMA_63BIT_PFN - 1, 0);
1020 if (!iova_ptr)
1021 err = -ENOMEM;
1022 }
1023 if (!err)
1024 *out_offset = (iova_ptr->pfn_lo) << PAGE_SHIFT;
1025 return err;
1026}
1027
1028/**
1029 * scif_free_window_offset:
1030 * @ep: end point descriptor
1031 * @window: registration window
1032 * @offset: Offset to be freed
1033 *
1034 * Free offset for this EP. The callee is supposed to grab
1035 * the RMA mutex before calling this API.
1036 */
1037void scif_free_window_offset(struct scif_endpt *ep,
1038 struct scif_window *window, s64 offset)
1039{
1040 if ((window && !window->offset_freed) || !window) {
1041 free_iova(&ep->rma_info.iovad, offset >> PAGE_SHIFT);
1042 if (window)
1043 window->offset_freed = true;
1044 }
1045}
1046
1047/**
1048 * scif_alloc_req: Respond to SCIF_ALLOC_REQ interrupt message
1049 * @msg: Interrupt message
1050 *
1051 * Remote side is requesting a memory allocation.
1052 */
1053void scif_alloc_req(struct scif_dev *scifdev, struct scifmsg *msg)
1054{
1055 int err;
1056 struct scif_window *window = NULL;
1057 int nr_pages = msg->payload[1];
1058
1059 window = scif_create_remote_window(scifdev, nr_pages);
1060 if (!window) {
1061 err = -ENOMEM;
1062 goto error;
1063 }
1064
1065 /* The peer's allocation request is granted */
1066 msg->uop = SCIF_ALLOC_GNT;
1067 msg->payload[0] = (u64)window;
1068 msg->payload[1] = window->mapped_offset;
1069 err = scif_nodeqp_send(scifdev, msg);
1070 if (err)
1071 scif_destroy_remote_window(window);
1072 return;
1073error:
1074 /* The peer's allocation request is rejected */
1075 dev_err(&scifdev->sdev->dev,
1076 "%s %d error %d alloc_ptr %p nr_pages 0x%x\n",
1077 __func__, __LINE__, err, window, nr_pages);
1078 msg->uop = SCIF_ALLOC_REJ;
1079 scif_nodeqp_send(scifdev, msg);
1080}
1081
1082/**
1083 * scif_alloc_gnt_rej: Respond to SCIF_ALLOC_GNT/REJ interrupt message
1084 * @msg: Interrupt message
1085 *
1086 * Remote side responded to a memory allocation.
1087 */
1088void scif_alloc_gnt_rej(struct scif_dev *scifdev, struct scifmsg *msg)
1089{
1090 struct scif_allocmsg *handle = (struct scif_allocmsg *)msg->payload[2];
1091 struct scif_window *window = container_of(handle, struct scif_window,
1092 alloc_handle);
1093 struct scif_endpt *ep = (struct scif_endpt *)window->ep;
1094
1095 mutex_lock(&ep->rma_info.rma_lock);
1096 handle->vaddr = msg->payload[0];
1097 handle->phys_addr = msg->payload[1];
1098 if (msg->uop == SCIF_ALLOC_GNT)
1099 handle->state = OP_COMPLETED;
1100 else
1101 handle->state = OP_FAILED;
1102 wake_up(&handle->allocwq);
1103 mutex_unlock(&ep->rma_info.rma_lock);
1104}
1105
1106/**
1107 * scif_free_virt: Respond to SCIF_FREE_VIRT interrupt message
1108 * @msg: Interrupt message
1109 *
1110 * Free up memory kmalloc'd earlier.
1111 */
1112void scif_free_virt(struct scif_dev *scifdev, struct scifmsg *msg)
1113{
1114 struct scif_window *window = (struct scif_window *)msg->payload[1];
1115
1116 scif_destroy_remote_window(window);
1117}
1118
1119static void
1120scif_fixup_aper_base(struct scif_dev *dev, struct scif_window *window)
1121{
1122 int j;
1123 struct scif_hw_dev *sdev = dev->sdev;
1124 phys_addr_t apt_base = 0;
1125
1126 /*
1127 * Add the aperture base if the DMA address is not card relative
1128 * since the DMA addresses need to be an offset into the bar
1129 */
1130 if (!scifdev_self(dev) && window->type == SCIF_WINDOW_PEER &&
1131 sdev->aper && !sdev->card_rel_da)
1132 apt_base = sdev->aper->pa;
1133 else
1134 return;
1135
1136 for (j = 0; j < window->nr_contig_chunks; j++) {
1137 if (window->num_pages[j])
1138 window->dma_addr[j] += apt_base;
1139 else
1140 break;
1141 }
1142}
1143
1144/**
1145 * scif_recv_reg: Respond to SCIF_REGISTER interrupt message
1146 * @msg: Interrupt message
1147 *
1148 * Update remote window list with a new registered window.
1149 */
1150void scif_recv_reg(struct scif_dev *scifdev, struct scifmsg *msg)
1151{
1152 struct scif_endpt *ep = (struct scif_endpt *)msg->payload[0];
1153 struct scif_window *window =
1154 (struct scif_window *)msg->payload[1];
1155
1156 mutex_lock(&ep->rma_info.rma_lock);
1157 spin_lock(&ep->lock);
1158 if (ep->state == SCIFEP_CONNECTED) {
1159 msg->uop = SCIF_REGISTER_ACK;
1160 scif_nodeqp_send(ep->remote_dev, msg);
1161 scif_fixup_aper_base(ep->remote_dev, window);
1162 /* No further failures expected. Insert new window */
1163 scif_insert_window(window, &ep->rma_info.remote_reg_list);
1164 } else {
1165 msg->uop = SCIF_REGISTER_NACK;
1166 scif_nodeqp_send(ep->remote_dev, msg);
1167 }
1168 spin_unlock(&ep->lock);
1169 mutex_unlock(&ep->rma_info.rma_lock);
1170 /* free up any lookup resources now that page lists are transferred */
1171 scif_destroy_remote_lookup(ep->remote_dev, window);
1172 /*
1173 * We could not insert the window but we need to
1174 * destroy the window.
1175 */
1176 if (msg->uop == SCIF_REGISTER_NACK)
1177 scif_destroy_remote_window(window);
1178}
1179
1180/**
1181 * scif_recv_unreg: Respond to SCIF_UNREGISTER interrupt message
1182 * @msg: Interrupt message
1183 *
1184 * Remove window from remote registration list;
1185 */
1186void scif_recv_unreg(struct scif_dev *scifdev, struct scifmsg *msg)
1187{
1188 struct scif_rma_req req;
1189 struct scif_window *window = NULL;
1190 struct scif_window *recv_window =
1191 (struct scif_window *)msg->payload[0];
1192 struct scif_endpt *ep;
1193 int del_window = 0;
1194
1195 ep = (struct scif_endpt *)recv_window->ep;
1196 req.out_window = &window;
1197 req.offset = recv_window->offset;
1198 req.prot = 0;
1199 req.nr_bytes = recv_window->nr_pages << PAGE_SHIFT;
1200 req.type = SCIF_WINDOW_FULL;
1201 req.head = &ep->rma_info.remote_reg_list;
1202 msg->payload[0] = ep->remote_ep;
1203
1204 mutex_lock(&ep->rma_info.rma_lock);
1205 /* Does a valid window exist? */
1206 if (scif_query_window(&req)) {
1207 dev_err(&scifdev->sdev->dev,
1208 "%s %d -ENXIO\n", __func__, __LINE__);
1209 msg->uop = SCIF_UNREGISTER_ACK;
1210 goto error;
1211 }
1212 if (window) {
1213 if (window->ref_count)
1214 scif_put_window(window, window->nr_pages);
1215 else
1216 dev_err(&scifdev->sdev->dev,
1217 "%s %d ref count should be +ve\n",
1218 __func__, __LINE__);
1219 window->unreg_state = OP_COMPLETED;
1220 if (!window->ref_count) {
1221 msg->uop = SCIF_UNREGISTER_ACK;
1222 atomic_inc(&ep->rma_info.tw_refcount);
1223 ep->rma_info.async_list_del = 1;
1224 list_del_init(&window->list);
1225 del_window = 1;
1226 } else {
1227 /* NACK! There are valid references to this window */
1228 msg->uop = SCIF_UNREGISTER_NACK;
1229 }
1230 } else {
1231 /* The window did not make its way to the list at all. ACK */
1232 msg->uop = SCIF_UNREGISTER_ACK;
1233 scif_destroy_remote_window(recv_window);
1234 }
1235error:
1236 mutex_unlock(&ep->rma_info.rma_lock);
1237 if (del_window)
1238 scif_drain_dma_intr(ep->remote_dev->sdev,
1239 ep->rma_info.dma_chan);
1240 scif_nodeqp_send(ep->remote_dev, msg);
1241 if (del_window)
1242 scif_queue_for_cleanup(window, &scif_info.rma);
1243}
1244
1245/**
1246 * scif_recv_reg_ack: Respond to SCIF_REGISTER_ACK interrupt message
1247 * @msg: Interrupt message
1248 *
1249 * Wake up the window waiting to complete registration.
1250 */
1251void scif_recv_reg_ack(struct scif_dev *scifdev, struct scifmsg *msg)
1252{
1253 struct scif_window *window =
1254 (struct scif_window *)msg->payload[2];
1255 struct scif_endpt *ep = (struct scif_endpt *)window->ep;
1256
1257 mutex_lock(&ep->rma_info.rma_lock);
1258 window->reg_state = OP_COMPLETED;
1259 wake_up(&window->regwq);
1260 mutex_unlock(&ep->rma_info.rma_lock);
1261}
1262
1263/**
1264 * scif_recv_reg_nack: Respond to SCIF_REGISTER_NACK interrupt message
1265 * @msg: Interrupt message
1266 *
1267 * Wake up the window waiting to inform it that registration
1268 * cannot be completed.
1269 */
1270void scif_recv_reg_nack(struct scif_dev *scifdev, struct scifmsg *msg)
1271{
1272 struct scif_window *window =
1273 (struct scif_window *)msg->payload[2];
1274 struct scif_endpt *ep = (struct scif_endpt *)window->ep;
1275
1276 mutex_lock(&ep->rma_info.rma_lock);
1277 window->reg_state = OP_FAILED;
1278 wake_up(&window->regwq);
1279 mutex_unlock(&ep->rma_info.rma_lock);
1280}
1281
1282/**
1283 * scif_recv_unreg_ack: Respond to SCIF_UNREGISTER_ACK interrupt message
1284 * @msg: Interrupt message
1285 *
1286 * Wake up the window waiting to complete unregistration.
1287 */
1288void scif_recv_unreg_ack(struct scif_dev *scifdev, struct scifmsg *msg)
1289{
1290 struct scif_window *window =
1291 (struct scif_window *)msg->payload[1];
1292 struct scif_endpt *ep = (struct scif_endpt *)window->ep;
1293
1294 mutex_lock(&ep->rma_info.rma_lock);
1295 window->unreg_state = OP_COMPLETED;
1296 wake_up(&window->unregwq);
1297 mutex_unlock(&ep->rma_info.rma_lock);
1298}
1299
1300/**
1301 * scif_recv_unreg_nack: Respond to SCIF_UNREGISTER_NACK interrupt message
1302 * @msg: Interrupt message
1303 *
1304 * Wake up the window waiting to inform it that unregistration
1305 * cannot be completed immediately.
1306 */
1307void scif_recv_unreg_nack(struct scif_dev *scifdev, struct scifmsg *msg)
1308{
1309 struct scif_window *window =
1310 (struct scif_window *)msg->payload[1];
1311 struct scif_endpt *ep = (struct scif_endpt *)window->ep;
1312
1313 mutex_lock(&ep->rma_info.rma_lock);
1314 window->unreg_state = OP_FAILED;
1315 wake_up(&window->unregwq);
1316 mutex_unlock(&ep->rma_info.rma_lock);
1317}
1318
1319int __scif_pin_pages(void *addr, size_t len, int *out_prot,
1320 int map_flags, scif_pinned_pages_t *pages)
1321{
1322 struct scif_pinned_pages *pinned_pages;
1323 int nr_pages, err = 0, i;
1324 bool vmalloc_addr = false;
1325 bool try_upgrade = false;
1326 int prot = *out_prot;
1327 int ulimit = 0;
1328 struct mm_struct *mm = NULL;
1329
1330 /* Unsupported flags */
1331 if (map_flags & ~(SCIF_MAP_KERNEL | SCIF_MAP_ULIMIT))
1332 return -EINVAL;
1333 ulimit = !!(map_flags & SCIF_MAP_ULIMIT);
1334
1335 /* Unsupported protection requested */
1336 if (prot & ~(SCIF_PROT_READ | SCIF_PROT_WRITE))
1337 return -EINVAL;
1338
1339 /* addr/len must be page aligned. len should be non zero */
1340 if (!len ||
1341 (ALIGN((u64)addr, PAGE_SIZE) != (u64)addr) ||
1342 (ALIGN((u64)len, PAGE_SIZE) != (u64)len))
1343 return -EINVAL;
1344
1345 might_sleep();
1346
1347 nr_pages = len >> PAGE_SHIFT;
1348
1349 /* Allocate a set of pinned pages */
1350 pinned_pages = scif_create_pinned_pages(nr_pages, prot);
1351 if (!pinned_pages)
1352 return -ENOMEM;
1353
1354 if (map_flags & SCIF_MAP_KERNEL) {
1355 if (is_vmalloc_addr(addr))
1356 vmalloc_addr = true;
1357
1358 for (i = 0; i < nr_pages; i++) {
1359 if (vmalloc_addr)
1360 pinned_pages->pages[i] =
1361 vmalloc_to_page(addr + (i * PAGE_SIZE));
1362 else
1363 pinned_pages->pages[i] =
1364 virt_to_page(addr + (i * PAGE_SIZE));
1365 }
1366 pinned_pages->nr_pages = nr_pages;
1367 pinned_pages->map_flags = SCIF_MAP_KERNEL;
1368 } else {
1369 /*
1370 * SCIF supports registration caching. If a registration has
1371 * been requested with read only permissions, then we try
1372 * to pin the pages with RW permissions so that a subsequent
1373 * transfer with RW permission can hit the cache instead of
1374 * invalidating it. If the upgrade fails with RW then we
1375 * revert back to R permission and retry
1376 */
1377 if (prot == SCIF_PROT_READ)
1378 try_upgrade = true;
1379 prot |= SCIF_PROT_WRITE;
1380retry:
1381 mm = current->mm;
1382 down_write(&mm->mmap_sem);
1383 if (ulimit) {
1384 err = __scif_check_inc_pinned_vm(mm, nr_pages);
1385 if (err) {
1386 up_write(&mm->mmap_sem);
1387 pinned_pages->nr_pages = 0;
1388 goto error_unmap;
1389 }
1390 }
1391
1392 pinned_pages->nr_pages = get_user_pages(
1393 current,
1394 mm,
1395 (u64)addr,
1396 nr_pages,
1397 !!(prot & SCIF_PROT_WRITE),
1398 0,
1399 pinned_pages->pages,
1400 NULL);
1401 up_write(&mm->mmap_sem);
1402 if (nr_pages != pinned_pages->nr_pages) {
1403 if (try_upgrade) {
1404 if (ulimit)
1405 __scif_dec_pinned_vm_lock(mm,
1406 nr_pages, 0);
1407 /* Roll back any pinned pages */
1408 for (i = 0; i < pinned_pages->nr_pages; i++) {
1409 if (pinned_pages->pages[i])
1410 put_page(
1411 pinned_pages->pages[i]);
1412 }
1413 prot &= ~SCIF_PROT_WRITE;
1414 try_upgrade = false;
1415 goto retry;
1416 }
1417 }
1418 pinned_pages->map_flags = 0;
1419 }
1420
1421 if (pinned_pages->nr_pages < nr_pages) {
1422 err = -EFAULT;
1423 pinned_pages->nr_pages = nr_pages;
1424 goto dec_pinned;
1425 }
1426
1427 *out_prot = prot;
1428 atomic_set(&pinned_pages->ref_count, 1);
1429 *pages = pinned_pages;
1430 return err;
1431dec_pinned:
1432 if (ulimit)
1433 __scif_dec_pinned_vm_lock(mm, nr_pages, 0);
1434 /* Something went wrong! Rollback */
1435error_unmap:
1436 pinned_pages->nr_pages = nr_pages;
1437 scif_destroy_pinned_pages(pinned_pages);
1438 *pages = NULL;
1439 dev_dbg(scif_info.mdev.this_device,
1440 "%s %d err %d len 0x%lx\n", __func__, __LINE__, err, len);
1441 return err;
1442}
1443
1444int scif_pin_pages(void *addr, size_t len, int prot,
1445 int map_flags, scif_pinned_pages_t *pages)
1446{
1447 return __scif_pin_pages(addr, len, &prot, map_flags, pages);
1448}
1449EXPORT_SYMBOL_GPL(scif_pin_pages);
1450
1451int scif_unpin_pages(scif_pinned_pages_t pinned_pages)
1452{
1453 int err = 0, ret;
1454
1455 if (!pinned_pages || SCIFEP_MAGIC != pinned_pages->magic)
1456 return -EINVAL;
1457
1458 ret = atomic_sub_return(1, &pinned_pages->ref_count);
1459 if (ret < 0) {
1460 dev_err(scif_info.mdev.this_device,
1461 "%s %d scif_unpin_pages called without pinning? rc %d\n",
1462 __func__, __LINE__, ret);
1463 return -EINVAL;
1464 }
1465 /*
1466 * Destroy the window if the ref count for this set of pinned
1467 * pages has dropped to zero. If it is positive then there is
1468 * a valid registered window which is backed by these pages and
1469 * it will be destroyed once all such windows are unregistered.
1470 */
1471 if (!ret)
1472 err = scif_destroy_pinned_pages(pinned_pages);
1473
1474 return err;
1475}
1476EXPORT_SYMBOL_GPL(scif_unpin_pages);
1477
1478static inline void
1479scif_insert_local_window(struct scif_window *window, struct scif_endpt *ep)
1480{
1481 mutex_lock(&ep->rma_info.rma_lock);
1482 scif_insert_window(window, &ep->rma_info.reg_list);
1483 mutex_unlock(&ep->rma_info.rma_lock);
1484}
1485
1486off_t scif_register_pinned_pages(scif_epd_t epd,
1487 scif_pinned_pages_t pinned_pages,
1488 off_t offset, int map_flags)
1489{
1490 struct scif_endpt *ep = (struct scif_endpt *)epd;
1491 s64 computed_offset;
1492 struct scif_window *window;
1493 int err;
1494 size_t len;
1495 struct device *spdev;
1496
1497 /* Unsupported flags */
1498 if (map_flags & ~SCIF_MAP_FIXED)
1499 return -EINVAL;
1500
1501 len = pinned_pages->nr_pages << PAGE_SHIFT;
1502
1503 /*
1504 * Offset is not page aligned/negative or offset+len
1505 * wraps around with SCIF_MAP_FIXED.
1506 */
1507 if ((map_flags & SCIF_MAP_FIXED) &&
1508 ((ALIGN(offset, PAGE_SIZE) != offset) ||
1509 (offset < 0) ||
1510 (offset + (off_t)len < offset)))
1511 return -EINVAL;
1512
1513 might_sleep();
1514
1515 err = scif_verify_epd(ep);
1516 if (err)
1517 return err;
1518 /*
1519 * It is an error to pass pinned_pages to scif_register_pinned_pages()
1520 * after calling scif_unpin_pages().
1521 */
1522 if (!atomic_add_unless(&pinned_pages->ref_count, 1, 0))
1523 return -EINVAL;
1524
1525 /* Compute the offset for this registration */
1526 err = scif_get_window_offset(ep, map_flags, offset,
1527 len, &computed_offset);
1528 if (err) {
1529 atomic_sub(1, &pinned_pages->ref_count);
1530 return err;
1531 }
1532
1533 /* Allocate and prepare self registration window */
1534 window = scif_create_window(ep, pinned_pages->nr_pages,
1535 computed_offset, false);
1536 if (!window) {
1537 atomic_sub(1, &pinned_pages->ref_count);
1538 scif_free_window_offset(ep, NULL, computed_offset);
1539 return -ENOMEM;
1540 }
1541
1542 window->pinned_pages = pinned_pages;
1543 window->nr_pages = pinned_pages->nr_pages;
1544 window->prot = pinned_pages->prot;
1545
1546 spdev = scif_get_peer_dev(ep->remote_dev);
1547 if (IS_ERR(spdev)) {
1548 err = PTR_ERR(spdev);
1549 scif_destroy_window(ep, window);
1550 return err;
1551 }
1552 err = scif_send_alloc_request(ep, window);
1553 if (err) {
1554 dev_err(&ep->remote_dev->sdev->dev,
1555 "%s %d err %d\n", __func__, __LINE__, err);
1556 goto error_unmap;
1557 }
1558
1559 /* Prepare the remote registration window */
1560 err = scif_prep_remote_window(ep, window);
1561 if (err) {
1562 dev_err(&ep->remote_dev->sdev->dev,
1563 "%s %d err %d\n", __func__, __LINE__, err);
1564 goto error_unmap;
1565 }
1566
1567 /* Tell the peer about the new window */
1568 err = scif_send_scif_register(ep, window);
1569 if (err) {
1570 dev_err(&ep->remote_dev->sdev->dev,
1571 "%s %d err %d\n", __func__, __LINE__, err);
1572 goto error_unmap;
1573 }
1574
1575 scif_put_peer_dev(spdev);
1576 /* No further failures expected. Insert new window */
1577 scif_insert_local_window(window, ep);
1578 return computed_offset;
1579error_unmap:
1580 scif_destroy_window(ep, window);
1581 scif_put_peer_dev(spdev);
1582 dev_err(&ep->remote_dev->sdev->dev,
1583 "%s %d err %d\n", __func__, __LINE__, err);
1584 return err;
1585}
1586EXPORT_SYMBOL_GPL(scif_register_pinned_pages);
1587
1588off_t scif_register(scif_epd_t epd, void *addr, size_t len, off_t offset,
1589 int prot, int map_flags)
1590{
1591 scif_pinned_pages_t pinned_pages;
1592 off_t err;
1593 struct scif_endpt *ep = (struct scif_endpt *)epd;
1594 s64 computed_offset;
1595 struct scif_window *window;
1596 struct mm_struct *mm = NULL;
1597 struct device *spdev;
1598
1599 dev_dbg(scif_info.mdev.this_device,
1600 "SCIFAPI register: ep %p addr %p len 0x%lx offset 0x%lx prot 0x%x map_flags 0x%x\n",
1601 epd, addr, len, offset, prot, map_flags);
1602 /* Unsupported flags */
1603 if (map_flags & ~(SCIF_MAP_FIXED | SCIF_MAP_KERNEL))
1604 return -EINVAL;
1605
1606 /*
1607 * Offset is not page aligned/negative or offset+len
1608 * wraps around with SCIF_MAP_FIXED.
1609 */
1610 if ((map_flags & SCIF_MAP_FIXED) &&
1611 ((ALIGN(offset, PAGE_SIZE) != offset) ||
1612 (offset < 0) ||
1613 (offset + (off_t)len < offset)))
1614 return -EINVAL;
1615
1616 /* Unsupported protection requested */
1617 if (prot & ~(SCIF_PROT_READ | SCIF_PROT_WRITE))
1618 return -EINVAL;
1619
1620 /* addr/len must be page aligned. len should be non zero */
1621 if (!len || (ALIGN((u64)addr, PAGE_SIZE) != (u64)addr) ||
1622 (ALIGN(len, PAGE_SIZE) != len))
1623 return -EINVAL;
1624
1625 might_sleep();
1626
1627 err = scif_verify_epd(ep);
1628 if (err)
1629 return err;
1630
1631 /* Compute the offset for this registration */
1632 err = scif_get_window_offset(ep, map_flags, offset,
1633 len >> PAGE_SHIFT, &computed_offset);
1634 if (err)
1635 return err;
1636
1637 spdev = scif_get_peer_dev(ep->remote_dev);
1638 if (IS_ERR(spdev)) {
1639 err = PTR_ERR(spdev);
1640 scif_free_window_offset(ep, NULL, computed_offset);
1641 return err;
1642 }
1643 /* Allocate and prepare self registration window */
1644 window = scif_create_window(ep, len >> PAGE_SHIFT,
1645 computed_offset, false);
1646 if (!window) {
1647 scif_free_window_offset(ep, NULL, computed_offset);
1648 scif_put_peer_dev(spdev);
1649 return -ENOMEM;
1650 }
1651
1652 window->nr_pages = len >> PAGE_SHIFT;
1653
1654 err = scif_send_alloc_request(ep, window);
1655 if (err) {
1656 scif_destroy_incomplete_window(ep, window);
1657 scif_put_peer_dev(spdev);
1658 return err;
1659 }
1660
1661 if (!(map_flags & SCIF_MAP_KERNEL)) {
1662 mm = __scif_acquire_mm();
1663 map_flags |= SCIF_MAP_ULIMIT;
1664 }
1665 /* Pin down the pages */
1666 err = __scif_pin_pages(addr, len, &prot,
1667 map_flags & (SCIF_MAP_KERNEL | SCIF_MAP_ULIMIT),
1668 &pinned_pages);
1669 if (err) {
1670 scif_destroy_incomplete_window(ep, window);
1671 __scif_release_mm(mm);
1672 goto error;
1673 }
1674
1675 window->pinned_pages = pinned_pages;
1676 window->prot = pinned_pages->prot;
1677 window->mm = mm;
1678
1679 /* Prepare the remote registration window */
1680 err = scif_prep_remote_window(ep, window);
1681 if (err) {
1682 dev_err(&ep->remote_dev->sdev->dev,
1683 "%s %d err %ld\n", __func__, __LINE__, err);
1684 goto error_unmap;
1685 }
1686
1687 /* Tell the peer about the new window */
1688 err = scif_send_scif_register(ep, window);
1689 if (err) {
1690 dev_err(&ep->remote_dev->sdev->dev,
1691 "%s %d err %ld\n", __func__, __LINE__, err);
1692 goto error_unmap;
1693 }
1694
1695 scif_put_peer_dev(spdev);
1696 /* No further failures expected. Insert new window */
1697 scif_insert_local_window(window, ep);
1698 dev_dbg(&ep->remote_dev->sdev->dev,
1699 "SCIFAPI register: ep %p addr %p len 0x%lx computed_offset 0x%llx\n",
1700 epd, addr, len, computed_offset);
1701 return computed_offset;
1702error_unmap:
1703 scif_destroy_window(ep, window);
1704error:
1705 scif_put_peer_dev(spdev);
1706 dev_err(&ep->remote_dev->sdev->dev,
1707 "%s %d err %ld\n", __func__, __LINE__, err);
1708 return err;
1709}
1710EXPORT_SYMBOL_GPL(scif_register);
1711
1712int
1713scif_unregister(scif_epd_t epd, off_t offset, size_t len)
1714{
1715 struct scif_endpt *ep = (struct scif_endpt *)epd;
1716 struct scif_window *window = NULL;
1717 struct scif_rma_req req;
1718 int nr_pages, err;
1719 struct device *spdev;
1720
1721 dev_dbg(scif_info.mdev.this_device,
1722 "SCIFAPI unregister: ep %p offset 0x%lx len 0x%lx\n",
1723 ep, offset, len);
1724 /* len must be page aligned. len should be non zero */
1725 if (!len ||
1726 (ALIGN((u64)len, PAGE_SIZE) != (u64)len))
1727 return -EINVAL;
1728
1729 /* Offset is not page aligned or offset+len wraps around */
1730 if ((ALIGN(offset, PAGE_SIZE) != offset) ||
1731 (offset + (off_t)len < offset))
1732 return -EINVAL;
1733
1734 err = scif_verify_epd(ep);
1735 if (err)
1736 return err;
1737
1738 might_sleep();
1739 nr_pages = len >> PAGE_SHIFT;
1740
1741 req.out_window = &window;
1742 req.offset = offset;
1743 req.prot = 0;
1744 req.nr_bytes = len;
1745 req.type = SCIF_WINDOW_FULL;
1746 req.head = &ep->rma_info.reg_list;
1747
1748 spdev = scif_get_peer_dev(ep->remote_dev);
1749 if (IS_ERR(spdev)) {
1750 err = PTR_ERR(spdev);
1751 return err;
1752 }
1753 mutex_lock(&ep->rma_info.rma_lock);
1754 /* Does a valid window exist? */
1755 err = scif_query_window(&req);
1756 if (err) {
1757 dev_err(&ep->remote_dev->sdev->dev,
1758 "%s %d err %d\n", __func__, __LINE__, err);
1759 goto error;
1760 }
1761 /* Unregister all the windows in this range */
1762 err = scif_rma_list_unregister(window, offset, nr_pages);
1763 if (err)
1764 dev_err(&ep->remote_dev->sdev->dev,
1765 "%s %d err %d\n", __func__, __LINE__, err);
1766error:
1767 mutex_unlock(&ep->rma_info.rma_lock);
1768 scif_put_peer_dev(spdev);
1769 return err;
1770}
1771EXPORT_SYMBOL_GPL(scif_unregister);